Merge pull request #32 from legop3/FEAT-announcer-audio-stream

Feat announcer audio stream
This commit is contained in:
legop3
2026-10-09 14:42:52 -04:00
committed by GitHub
42 changed files with 761 additions and 226 deletions
+1
View File
@@ -5,6 +5,7 @@
.git .git
.gitignore .gitignore
**/node_modules **/node_modules
webui/.cache/mediamtx/
# Neither the legacy state directory nor the Compose-mounted replacement may # Neither the legacy state directory nor the Compose-mounted replacement may
# enter an image build. This prevents credentials and backups from becoming # enter an image build. This prevents credentials and backups from becoming
# image layers after an operator has started using either deployment layout. # image layers after an operator has started using either deployment layout.
+1
View File
@@ -5,6 +5,7 @@ create_2_Open_Interface_Spec.txt
logs logs
node_modules/ node_modules/
webui/.cache/mediamtx/
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
.pio .pio
+5 -1
View File
@@ -23,9 +23,13 @@ services:
# Required for Bluetooth hardware such as the Balance Board. # Required for Bluetooth hardware such as the Balance Board.
- /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro
# Required for Kinect USB access. # Kinect USB access.
devices: devices:
- /dev/bus/usb:/dev/bus/usb - /dev/bus/usb:/dev/bus/usb
# For a server-local microphone, uncomment this device and group_add below.
# - /dev/snd:/dev/snd
# group_add:
# - "${AUDIO_GID:?Set AUDIO_GID to the host sound device group ID}"
cap_add: cap_add:
- NET_ADMIN - NET_ADMIN
+2
View File
@@ -13,6 +13,7 @@ const audioForward = require('../services/audioForwardService/configuration');
const audioLevels = require('../services/audioLevelsService/configuration'); const audioLevels = require('../services/audioLevelsService/configuration');
const homeAssistant = require('../services/homeAssistantService/configuration'); const homeAssistant = require('../services/homeAssistantService/configuration');
const homeAssistantActivities = require('../services/homeAssistantActivitiesService/configuration'); const homeAssistantActivities = require('../services/homeAssistantActivitiesService/configuration');
const roomAudio = require('../services/roomAudioService/configuration');
const roomCameras = require('../services/roomCameraService/configuration'); const roomCameras = require('../services/roomCameraService/configuration');
const ptzCamera = require('../services/ptzCameraService/configuration'); const ptzCamera = require('../services/ptzCameraService/configuration');
const kinect = require('../services/kinectService/configuration'); const kinect = require('../services/kinectService/configuration');
@@ -46,6 +47,7 @@ const definitions = [
// Activities own a separate catalog even though the HA connection is shared. // Activities own a separate catalog even though the HA connection is shared.
homeAssistantActivities, homeAssistantActivities,
roomCameras, roomCameras,
roomAudio,
ptzCamera, ptzCamera,
kinect, kinect,
balanceBoard, balanceBoard,
+2 -1
View File
@@ -124,7 +124,7 @@ function grantExternalSpectatorAccessAfterAdminLogin(socket) {
} }
io.on('connection', (socket) => { io.on('connection', (socket) => {
const requestedRole = socket.handshake?.query?.role; const requestedRole = socket.handshake?.auth?.role || socket.handshake?.query?.role;
/* /*
Role is assigned before the browser's full identity heartbeat has completed. Role is assigned before the browser's full identity heartbeat has completed.
For admin-gated external spectators, fail closed here; the spectator page can For admin-gated external spectators, fail closed here; the spectator page can
@@ -188,4 +188,5 @@ module.exports = {
isAdmin, isAdmin,
isLockdownAdmin, isLockdownAdmin,
authenticate, authenticate,
canBecomeSpectator,
}; };
@@ -91,9 +91,9 @@ function createWebReplayTextCommand(socket, sendSystemMessage, replayApi) {
const requester = buildRequesterLabel(socket); const requester = buildRequesterLabel(socket);
const jobId = buildReplayJobId('web-chat'); const jobId = buildReplayJobId('web-chat');
const attempt = replayApi.tryTriggerReplay({ by: { source: 'web-chat', requester } }); const attempt = replayApi.tryTriggerReplay({ jobId, by: { source: 'web-chat', requester } });
if (!attempt.ok) { if (!attempt.ok) {
await message.reply({ content: `Replay denied: cooldown active. Try again in ${Math.ceil(attempt.remainingMs / 1000)}s.` }); await message.reply({ content: `Replay denied: ${attempt.error}` });
return; return;
} }
@@ -16,7 +16,7 @@ const { renderIndexHtml, renderOgImage, renderWebManifest } = require('../embedS
a direct browser load. Keeping the setup and admin routes in this explicit a direct browser load. Keeping the setup and admin routes in this explicit
allowlist prevents them from working only after client-side navigation. allowlist prevents them from working only after client-side navigation.
*/ */
app.get(['/', '/old', '/spectate', '/mini', '/display', '/scanner', '/database', '/ptz', '/reports', '/setup', '/admin'], async (req, res) => { app.get(['/', '/old', '/spectate', '/mini', '/display', '/scanner', '/database', '/ptz', '/reports', '/setup', '/admin', '/mic'], async (req, res) => {
try { try {
const html = await renderIndexHtml(req); const html = await renderIndexHtml(req);
res.type('html').send(html); res.type('html').send(html);
@@ -20,6 +20,7 @@ function normalizeAdditionalHosts(rawHosts) {
function buildMediaMtxConfig({ config, serverPort, snapshotWriterPath }) { function buildMediaMtxConfig({ config, serverPort, snapshotWriterPath }) {
const media = config?.media || {}; const media = config?.media || {};
const webrtcPort = media.webrtcPort ?? 8189;
const configuredHosts = normalizeAdditionalHosts(media.additionalHosts); const configuredHosts = normalizeAdditionalHosts(media.additionalHosts);
let publicHostname = ''; let publicHostname = '';
try { try {
@@ -55,10 +56,10 @@ function buildMediaMtxConfig({ config, serverPort, snapshotWriterPath }) {
webrtc: true, webrtc: true,
// WHEP and WHIP signaling is public only through the Node /video proxy. // WHEP and WHIP signaling is public only through the Node /video proxy.
// ICE transport on 8189 remains directly reachable by browsers. // The configured ICE transport port remains directly reachable by browsers.
webrtcAddress: '127.0.0.1:8889', webrtcAddress: '127.0.0.1:8889',
webrtcLocalUDPAddress: ':8189', webrtcLocalUDPAddress: `:${webrtcPort}`,
webrtcLocalTCPAddress: ':8189', webrtcLocalTCPAddress: `:${webrtcPort}`,
webrtcAdditionalHosts: additionalHosts, webrtcAdditionalHosts: additionalHosts,
webrtcICEServers2: [ webrtcICEServers2: [
{ url: 'stun:stun.l.google.com:19302' }, { url: 'stun:stun.l.google.com:19302' },
@@ -81,6 +82,7 @@ function buildMediaMtxConfig({ config, serverPort, snapshotWriterPath }) {
{ action: 'pprof' }, { action: 'pprof' },
], ],
paths: { paths: {
'~^room-audio/': { source: 'publisher' },
all: { all: {
source: 'publisher', source: 'publisher',
sourceOnDemand: false, sourceOnDemand: false,
@@ -1,18 +1,24 @@
// Media Transport Configuration // Media Transport Configuration
// Purpose: Defines additional MediaMTX ICE hosts not already derived from the server's public URL. // Purpose: Defines the WebRTC media port and additional ICE hosts.
// Scope: Contains configuration metadata only and never starts MediaMTX. // Scope: Contains configuration metadata only and never starts MediaMTX.
const { strictObject, stringArray } = require('../../configuration/schemaHelpers'); const { strictObject, stringArray, integer } = require('../../configuration/schemaHelpers');
module.exports = { module.exports = {
key: 'media', key: 'media',
defaultValue: { defaultValue: {
webrtcPort: 8189,
additionalHosts: [], additionalHosts: [],
}, },
schema: strictObject({ schema: strictObject({
webrtcPort: integer({
title: 'WebRTC port',
description: 'UDP and TCP port for live media. Forward the same external port to this server; HTTPS signaling continues through the web UI.',
minimum: 1, maximum: 65535,
}),
additionalHosts: stringArray({ additionalHosts: stringArray({
title: 'Additional ICE hosts', title: 'Additional ICE hosts',
item: { description: 'Hostname or IP address MediaMTX advertises as a WebRTC ICE candidate.', examples: ['rover.example.com', 'media-server.local'], minLength: 1, maxLength: 255 }, item: { description: 'Hostname or IP address MediaMTX advertises as a WebRTC ICE candidate.', examples: ['rover.example.com', 'media-server.local'], minLength: 1, maxLength: 255 },
array: { description: 'Extra public or LAN hostnames and addresses browsers may use in addition to the hostname derived from the top-level public URL.', uniqueItems: true }, array: { description: 'Extra public or LAN hostnames and addresses browsers may use in addition to the hostname derived from the top-level public URL.', uniqueItems: true },
}), }),
}, { title: 'Media', description: 'Adds optional WebRTC network candidates to the public hostname and local interfaces generated automatically for MediaMTX.', required: ['additionalHosts'] }), }, { title: 'Media', description: 'Configures the WebRTC media port and additional network candidates.', required: ['webrtcPort', 'additionalHosts'] }),
}; };
+7 -1
View File
@@ -6,6 +6,7 @@ const { promisify } = require('util');
const fsp = require('fs/promises'); const fsp = require('fs/promises');
const logger = require('../../globals/logger').child('replayEngineV2'); const logger = require('../../globals/logger').child('replayEngineV2');
const roverManager = require('../roverManager'); const roverManager = require('../roverManager');
const { roomAudioEvents } = require('../roomAudioService');
const { getRoomCameras, roomCameraEvents } = require('../roomCameraService'); const { getRoomCameras, roomCameraEvents } = require('../roomCameraService');
const { const {
@@ -40,8 +41,8 @@ function getActiveSegmentRoot() {
return runtime.activeSegmentRoot; return runtime.activeSegmentRoot;
} }
const workerManager = createWorkerManager({ getActiveSegmentRoot });
const segmentStore = createSegmentStore({ getActiveSegmentRoot }); const segmentStore = createSegmentStore({ getActiveSegmentRoot });
const workerManager = createWorkerManager({ getActiveSegmentRoot, onWorkerClosed: segmentStore.refreshIndexForWorker });
const sidebarRenderer = createSidebarRenderer({ execFileAsync, ensureDir }); const sidebarRenderer = createSidebarRenderer({ execFileAsync, ensureDir });
const replayBuilder = createReplayBuilder({ const replayBuilder = createReplayBuilder({
execFileAsync, execFileAsync,
@@ -50,6 +51,7 @@ const replayBuilder = createReplayBuilder({
renderSidebarVideo: sidebarRenderer.renderSidebarVideo, renderSidebarVideo: sidebarRenderer.renderSidebarVideo,
getVideoEntriesForSource: segmentStore.getVideoEntriesForSource, getVideoEntriesForSource: segmentStore.getVideoEntriesForSource,
getAudioEntriesForSource: segmentStore.getAudioEntriesForSource, getAudioEntriesForSource: segmentStore.getAudioEntriesForSource,
getRoomAudioEntries: segmentStore.getRoomAudioEntries,
overlapping: segmentStore.overlapping, overlapping: segmentStore.overlapping,
}); });
registerReplaySocketHooks({ tryTriggerReplay, validateSources, getDefaultWebSources }); registerReplaySocketHooks({ tryTriggerReplay, validateSources, getDefaultWebSources });
@@ -88,6 +90,10 @@ async function start() {
}, CLEANUP_INTERVAL_MS); }, CLEANUP_INTERVAL_MS);
} }
roomAudioEvents.on('update', () => {
tick().catch((err) => logger.warn('room audio update tick failed', err.message));
});
roomCameraEvents.on('update', () => { roomCameraEvents.on('update', () => {
tick().catch((err) => logger.warn('room camera update tick failed', err.message)); tick().catch((err) => logger.warn('room camera update tick failed', err.message));
}); });
@@ -51,7 +51,7 @@ function buildChatEventsForWindow(startMs, endMs, limit = 22, preWindowCount = 1
return [...beforeWindow, ...inWindow].sort((a, b) => a.ts - b.ts); return [...beforeWindow, ...inWindow].sort((a, b) => a.ts - b.ts);
} }
function createReplayBuilder({ execFileAsync, fsp, ensureDir, renderSidebarVideo, getVideoEntriesForSource, getAudioEntriesForSource, overlapping }) { function createReplayBuilder({ execFileAsync, fsp, ensureDir, renderSidebarVideo, getVideoEntriesForSource, getAudioEntriesForSource, getRoomAudioEntries, overlapping }) {
function resolveReplayWindow({ sources = [], nowMs, guardMs, durationMs }) { function resolveReplayWindow({ sources = [], nowMs, guardMs, durationMs }) {
const tentativeEnd = nowMs - guardMs; const tentativeEnd = nowMs - guardMs;
const sourceEnds = []; const sourceEnds = [];
@@ -186,6 +186,28 @@ function createReplayBuilder({ execFileAsync, fsp, ensureDir, renderSidebarVideo
if (!normalizedVideos.length) throw new Error('No replay segments available for selected sources'); if (!normalizedVideos.length) throw new Error('No replay segments available for selected sources');
const roomAudioEntries = await pinSegmentFiles(
overlapping(getRoomAudioEntries(), tStart, tEnd), tmpDir, 'room-audio-seg',
);
if (roomAudioEntries.length) {
// Place each segment at its recorded time. A mic joining late or
// reconnecting must leave silence in the gap, not shift its speech earlier.
const audioInputs = roomAudioEntries.flatMap((entry) => ['-i', entry.filePath]);
const audioFilters = roomAudioEntries.map((entry, index) => {
const start = Math.max(0, (tStart - entry.startMs) / 1000);
const duration = (Math.min(tEnd, entry.endMs) - Math.max(tStart, entry.startMs)) / 1000;
const delay = Math.max(0, Math.round(entry.startMs - tStart));
return `[${index}:a]atrim=start=${start.toFixed(3)}:duration=${duration.toFixed(3)},asetpts=PTS-STARTPTS,adelay=${delay}:all=1[room${index}]`;
});
const refs = roomAudioEntries.map((_, index) => `[room${index}]`).join('');
audioFilters.push(`${refs}amix=inputs=${roomAudioEntries.length}:normalize=0[roomout]`);
const roomAudioPath = path.join(tmpDir, 'room-audio.m4a');
await execFileAsync(FFMPEG_BIN, ['-y', '-hide_banner', '-loglevel', 'error', ...audioInputs,
'-filter_complex', audioFilters.join(';'), '-map', '[roomout]', '-vn', '-ac', '1', '-ar', '48000',
'-c:a', 'aac', '-b:a', '96k', roomAudioPath]);
normalizedAudios.push(roomAudioPath);
}
const layout = buildGridLayout(normalizedVideos.length); const layout = buildGridLayout(normalizedVideos.length);
const { maxWidth, maxHeight } = await probeMaxFrameSize(normalizedVideos.map((v) => v.path)); const { maxWidth, maxHeight } = await probeMaxFrameSize(normalizedVideos.map((v) => v.path));
let tileWidth = maxWidth || 640; let tileWidth = maxWidth || 640;
@@ -6,9 +6,10 @@ const fs = require('fs');
const path = require('path'); const path = require('path');
const logger = require('../../globals/logger').child('replayEngineV2'); const logger = require('../../globals/logger').child('replayEngineV2');
const { BUFFER_SECONDS, SEGMENT_SECONDS } = require('./constants'); const { BUFFER_SECONDS, SEGMENT_SECONDS } = require('./constants');
const { workers, segmentIndex } = require('./state'); const { workers, pendingWorkerStarts, segmentIndex } = require('./state');
const { sourceKey, sourceDirForKey } = require('./sources'); const { sourceKey, sourceDirForKey } = require('./sources');
const ptzCameraService = require('../ptzCameraService'); const ptzCameraService = require('../ptzCameraService');
const SEGMENT_FILENAME_PATTERN = /^seg-[a-f0-9-]+-\d{6}\.mp4$/;
async function ensureDir(dir) { async function ensureDir(dir) {
await fsp.mkdir(dir, { recursive: true }); await fsp.mkdir(dir, { recursive: true });
@@ -25,7 +26,7 @@ function createSegmentStore({ getActiveSegmentRoot }) {
return; return;
} }
const segmentFiles = files.filter((entry) => entry.isFile() && /^seg-\d{6}\.mp4$/.test(entry.name)).map((entry) => entry.name).sort(); const segmentFiles = files.filter((entry) => entry.isFile() && SEGMENT_FILENAME_PATTERN.test(entry.name)).map((entry) => entry.name).sort();
if (!segmentFiles.length) { if (!segmentFiles.length) {
segmentIndex.set(key, []); segmentIndex.set(key, []);
return; return;
@@ -64,6 +65,13 @@ function createSegmentStore({ getActiveSegmentRoot }) {
async function cleanupOldFiles() { async function cleanupOldFiles() {
const cutoff = Date.now() - BUFFER_SECONDS * 1000; const cutoff = Date.now() - BUFFER_SECONDS * 1000;
// Retention applies to stopped workers too; dynamic microphone sessions
// must not leave permanent index entries after their audio expires.
for (const [key, entries] of segmentIndex) {
const retained = entries.filter((entry) => entry.endMs >= cutoff);
if (!retained.length && !workers.has(key) && !pendingWorkerStarts.has(key)) segmentIndex.delete(key);
else segmentIndex.set(key, retained);
}
const root = getActiveSegmentRoot(); const root = getActiveSegmentRoot();
try { try {
await ensureDir(root); await ensureDir(root);
@@ -81,6 +89,10 @@ function createSegmentStore({ getActiveSegmentRoot }) {
if (stat.mtimeMs < cutoff) await fsp.unlink(filePath); if (stat.mtimeMs < cutoff) await fsp.unlink(filePath);
} catch {} } catch {}
} }
if (!workers.has(dirent.name) && !pendingWorkerStarts.has(dirent.name)) {
// rmdir only removes empty directories, preserving buffered or newly written files.
try { await fsp.rmdir(dirPath); } catch {}
}
} }
} catch (err) { } catch (err) {
logger.warn('cleanup failed', err.message); logger.warn('cleanup failed', err.message);
@@ -115,6 +127,14 @@ function createSegmentStore({ getActiveSegmentRoot }) {
return []; return [];
} }
function getRoomAudioEntries() {
// Include retained buffers from microphones that disconnected during the
// replay window, rather than limiting history to currently live streams.
return [...segmentIndex.entries()]
.filter(([key]) => key.startsWith('roomAudio__audio__'))
.flatMap(([, entries]) => entries);
}
function overlapping(entries, startMs, endMs) { function overlapping(entries, startMs, endMs) {
return entries.filter((entry) => entry.endMs > startMs && entry.startMs < endMs); return entries.filter((entry) => entry.endMs > startMs && entry.startMs < endMs);
} }
@@ -153,7 +173,7 @@ function createSegmentStore({ getActiveSegmentRoot }) {
try { try {
const files = fs.readdirSync(dir); const files = fs.readdirSync(dir);
for (const name of files) { for (const name of files) {
if (!/^seg-\d{6}\.mp4$/.test(name)) continue; if (!SEGMENT_FILENAME_PATTERN.test(name)) continue;
const full = path.join(dir, name); const full = path.join(dir, name);
let stat; let stat;
try { stat = fs.statSync(full); } catch { continue; } try { stat = fs.statSync(full); } catch { continue; }
@@ -172,10 +192,12 @@ function createSegmentStore({ getActiveSegmentRoot }) {
return { return {
refreshSegmentIndex, refreshSegmentIndex,
refreshIndexForWorker,
cleanupOldFiles, cleanupOldFiles,
getVideoEntriesForSource, getVideoEntriesForSource,
getAudioEntriesForRover, getAudioEntriesForRover,
getAudioEntriesForSource, getAudioEntriesForSource,
getRoomAudioEntries,
overlapping, overlapping,
bootstrapIndexFromDisk, bootstrapIndexFromDisk,
getReplayHealthSnapshot, getReplayHealthSnapshot,
@@ -2,8 +2,10 @@
// Purpose: Resolves replay-capable media sources and ffmpeg worker arguments. // Purpose: Resolves replay-capable media sources and ffmpeg worker arguments.
// Scope: Converts live rover/room state into stable source descriptors and stream worker config. // Scope: Converts live rover/room state into stable source descriptors and stream worker config.
const path = require('path'); const path = require('path');
const { randomUUID } = require('crypto');
const roverManager = require('../roverManager'); const roverManager = require('../roverManager');
const { getRoomCameras } = require('../roomCameraService'); const { getRoomCameras } = require('../roomCameraService');
const { getActiveRoomAudioStreams } = require('../roomAudioService');
const ptzCameraService = require('../ptzCameraService'); const ptzCameraService = require('../ptzCameraService');
const { FFMPEG_BIN, SEGMENT_SECONDS, TARGET_FPS } = require('./constants'); const { FFMPEG_BIN, SEGMENT_SECONDS, TARGET_FPS } = require('./constants');
@@ -51,12 +53,17 @@ function listDesiredSources() {
// both come from the same live MediaMTX path, unlike rovers where audio is a // both come from the same live MediaMTX path, unlike rovers where audio is a
// separate published stream. // separate published stream.
sources.push(...ptzCameraService.getReplayWorkerSources()); sources.push(...ptzCameraService.getReplayWorkerSources());
for (const stream of getActiveRoomAudioStreams()) {
sources.push({ id: stream.id, sourceType: 'roomAudio', kind: 'audio', label: stream.name,
inputUrl: `rtsp://127.0.0.1:8554/room-audio/${encodeURIComponent(stream.id)}` });
}
return sources; return sources;
} }
function buildWorkerArgs(activeSegmentRoot, source) { function buildWorkerArgs(activeSegmentRoot, source) {
const dir = sourceDirForKey(activeSegmentRoot, sourceKey(source)); const dir = sourceDirForKey(activeSegmentRoot, sourceKey(source));
const pattern = path.join(dir, 'seg-%06d.mp4'); // Each capture process owns its filenames so restarts preserve buffered segments.
const pattern = path.join(dir, `seg-${randomUUID()}-%06d.mp4`);
// MediaMTX and RTSP cameras use TCP so replay capture has one reliable // MediaMTX and RTSP cameras use TCP so replay capture has one reliable
// transport and never falls back to separate RTP/RTCP UDP listeners. // transport and never falls back to separate RTP/RTCP UDP listeners.
const inputTransport = /^rtsps?:\/\//i.test(source.inputUrl) const inputTransport = /^rtsps?:\/\//i.test(source.inputUrl)
@@ -12,7 +12,7 @@ async function ensureDir(dir) {
await fsp.mkdir(dir, { recursive: true }); await fsp.mkdir(dir, { recursive: true });
} }
function createWorkerManager({ getActiveSegmentRoot }) { function createWorkerManager({ getActiveSegmentRoot, onWorkerClosed }) {
function startWorker(source) { function startWorker(source) {
const key = sourceKey(source); const key = sourceKey(source);
if (workers.has(key) || pendingWorkerStarts.has(key)) return; if (workers.has(key) || pendingWorkerStarts.has(key)) return;
@@ -21,26 +21,36 @@ function createWorkerManager({ getActiveSegmentRoot }) {
const dir = sourceDirForKey(getActiveSegmentRoot(), key); const dir = sourceDirForKey(getActiveSegmentRoot(), key);
ensureDir(dir) ensureDir(dir)
.then(() => { .then(() => {
if (workers.has(key)) { // Directory creation can finish after a microphone has disconnected.
if (workers.has(key) || !listDesiredSources().some((entry) => sourceKey(entry) === key)) {
pendingWorkerStarts.delete(key); pendingWorkerStarts.delete(key);
return; return;
} }
const proc = spawn(FFMPEG_BIN, args, { stdio: ['ignore', 'ignore', 'pipe'] }); const proc = spawn(FFMPEG_BIN, args, { stdio: ['ignore', 'ignore', 'pipe'] });
workers.set(key, { key, source, proc }); const worker = { key, source, proc, stopping: false };
workers.set(key, worker);
pendingWorkerStarts.delete(key); pendingWorkerStarts.delete(key);
proc.stderr.on('data', (chunk) => { proc.stderr.on('data', (chunk) => {
const text = String(chunk || '').trim(); const text = String(chunk || '').trim();
if (!text) return; if (!text) return;
logger.warn('worker stderr', { key, text: text.slice(0, 500) }); logger.warn('worker stderr', { key, text: text.slice(0, 500) });
}); });
proc.on('exit', (code, signal) => { proc.on('error', (err) => logger.warn('worker process failed', { key, error: err.message }));
const current = workers.get(key); worker.finished = new Promise((resolve) => {
if (current?.proc === proc) workers.delete(key); proc.once('close', async (code, signal) => {
logger.warn('worker exited', { key, code, signal }); clearTimeout(worker.killTimer);
setTimeout(() => { // FFmpeg writes the final MP4 trailer before close. Index it before
const desired = listDesiredSources().find((entry) => sourceKey(entry) === key); // releasing this worker so a disconnect cannot drop its last audio.
if (desired && !workers.has(key)) startWorker(desired); try { await onWorkerClosed(worker); }
}, 1500); catch (err) { logger.warn('final worker indexing failed', { key, error: err.message }); }
if (workers.get(key) === worker) workers.delete(key);
resolve();
logger.warn('worker exited', { key, code, signal });
if (!worker.stopping) setTimeout(() => {
const desired = listDesiredSources().find((entry) => sourceKey(entry) === key);
if (desired && !workers.has(key)) startWorker(desired);
}, 1500);
});
}); });
}) })
.catch((err) => { .catch((err) => {
@@ -52,8 +62,14 @@ function createWorkerManager({ getActiveSegmentRoot }) {
function stopWorker(key) { function stopWorker(key) {
const worker = workers.get(key); const worker = workers.get(key);
if (!worker) return; if (!worker) return;
try { worker.proc.kill('SIGTERM'); } catch {} if (!worker.stopping) {
workers.delete(key); worker.stopping = true;
worker.proc.kill('SIGTERM');
// Bound shutdown if a capture is blocked waiting on its network input.
worker.killTimer = setTimeout(() => worker.proc.kill('SIGKILL'), 3000);
worker.killTimer.unref();
}
return worker.finished;
} }
async function syncWorkers() { async function syncWorkers() {
@@ -63,9 +79,11 @@ function createWorkerManager({ getActiveSegmentRoot }) {
const key = sourceKey(source); const key = sourceKey(source);
if (!workers.has(key)) startWorker(source); if (!workers.has(key)) startWorker(source);
} }
const stops = [];
for (const key of Array.from(workers.keys())) { for (const key of Array.from(workers.keys())) {
if (!desiredKeys.has(key)) stopWorker(key); if (!desiredKeys.has(key)) stops.push(stopWorker(key));
} }
await Promise.all(stops);
} }
return { startWorker, stopWorker, syncWorkers }; return { startWorker, stopWorker, syncWorkers };
@@ -0,0 +1,14 @@
const { strictObject, string, boolean } = require('../../configuration/schemaHelpers');
module.exports = {
key: 'roomAudio', feature: true,
defaultValue: { enabled: false, sources: [] },
schema: strictObject({
enabled: boolean({ description: 'Enable room audio and local browser microphone publishing.' }),
sources: { type: 'array', items: strictObject({
id: string({ minLength: 1, maxLength: 80, pattern: '^[a-zA-Z0-9_-]+$' }),
name: string({ minLength: 1, maxLength: 120 }),
type: { type: 'string', enum: ['device', 'stream'] },
input: string({ minLength: 1, maxLength: 2048, description: 'ALSA device (for example default or plughw:CARD=USB,DEV=0), or a network stream URL.' }),
}, { required: ['id', 'name', 'type', 'input'] }) },
}, { title: 'Room audio', required: ['enabled', 'sources'] }),
};
@@ -0,0 +1,171 @@
const { spawn } = require('child_process');
const EventEmitter = require('events');
const io = require('../../globals/io');
const logger = require('../../globals/logger').child('roomAudio');
const { loadConfig, registerConfigurationHandler } = require('../../configuration');
const { getSocketIp, isLocalNetwork } = require('../../helpers/ipResolver');
const { getMode, MODES } = require('../modeManager');
const { getRole, isAdmin, isLockdownAdmin } = require('../roleService');
const { canBecomeSpectator } = require('../authService');
const { isDeterred } = require('../verificationService');
const videoSessions = require('../videoSessions');
const roomAudioEvents = new EventEmitter();
const sources = new Map();
const mediaSessions = new Map();
let enabled = false;
let active = [];
let polling = false;
let generation = 0;
function canListen(socket) {
if (!enabled || !socket) return false;
const mode = getMode();
if (mode === MODES.LOCKDOWN) return isLockdownAdmin(socket);
if (isAdmin(socket)) return true;
if (getRole(socket) === 'spectator') return canBecomeSpectator(socket);
// Room audio follows general driving eligibility, without tying listening
// to rover assignment, private-rover access, or the current driving turn.
return getRole(socket) === 'user' && mode !== MODES.ADMIN && !isDeterred(socket);
}
function canPublish(socket, id) {
return enabled && isLocalNetwork(getSocketIp(socket)) && sources.get(id)?.socketId === socket.id;
}
function getRoomAudioStreams(socket) { return canListen(socket) ? active : []; }
function getActiveRoomAudioStreams() { return active; }
function trackMediaSession(id, token) { if (id) mediaSessions.set(id, token); }
function hasStream(id) { return active.some((source) => source.id === id); }
function stopSource(id) {
const source = sources.get(id);
if (!source) return;
sources.delete(id);
clearTimeout(source.timer);
source.child?.kill('SIGTERM');
videoSessions.revokeWhere((info) => ['roomAudio', 'roomMic'].includes(info.sourceType) && info.sourceId === id);
active = active.filter((entry) => entry.id !== id);
roomAudioEvents.emit('update');
}
function startWorker(source) {
if (sources.get(source.id) !== source) return;
const args = ['-nostdin', '-hide_banner', '-loglevel', 'warning'];
if (source.type === 'device') args.push('-f', 'alsa');
else if (/^rtsps?:/i.test(source.input)) args.push('-rtsp_transport', 'tcp', '-timeout', '10000000');
else args.push('-rw_timeout', '10000000');
args.push('-i', source.input, '-map', '0:a:0', '-vn', '-c:a', 'libopus', '-ar', '48000', '-ac', '1', '-b:a', '64k', '-f', 'rtsp', '-rtsp_transport', 'tcp', `rtsp://127.0.0.1:8554/room-audio/${source.id}`);
const child = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] });
source.child = child;
child.stderr.on('data', (data) => logger.warn('Room audio %s: %s', source.name, String(data).trim()));
child.on('error', (error) => logger.warn('Room audio %s: %s', source.name, error.message));
child.on('close', () => {
if (sources.get(source.id) !== source) return;
source.child = null;
source.timer = setTimeout(() => startWorker(source), 2000);
});
}
function applyConfig(config = {}) {
const ids = (config.sources || []).map((source) => source.id);
if (new Set(ids).size !== ids.length) throw new Error('Room audio source IDs must be unique');
generation += 1;
enabled = Boolean(config.enabled);
const configured = new Map((config.sources || []).map((entry) => {
const source = { ...entry, id: `static-${entry.id}` };
return [source.id, source];
}));
for (const [id, source] of sources) {
const next = configured.get(id);
// Static-source edits should not interrupt browser microphones or restart
// unchanged captures. Disabling the feature still stops every source.
if (!enabled || (!source.socketId && (!next || next.type !== source.type || next.input !== source.input))) stopSource(id);
}
if (enabled) for (const [id, source] of configured) {
const existing = sources.get(id);
if (existing) existing.name = source.name;
else {
sources.set(id, source);
startWorker(source);
}
}
roomAudioEvents.emit('update');
}
async function listMediaMtxItems(collection) {
const items = [];
let page = 0;
let pageCount = 1;
while (page < pageCount) {
const response = await fetch(`http://127.0.0.1:9997/v3/${collection}/list?page=${page}`, { signal: AbortSignal.timeout(1500) });
if (!response.ok) throw new Error('MediaMTX unavailable');
const data = await response.json();
items.push(...(data.items || []));
pageCount = data.pageCount || 1;
page += 1;
}
return items;
}
// Advertise actual ready MediaMTX paths, rather than a browser reservation or
// an ffmpeg process that may still be trying to connect to its input.
async function refreshStreams() {
if ((!enabled && !mediaSessions.size) || polling) return;
polling = true;
const currentGeneration = generation;
let ready = new Set();
try {
// HTTP authorization is checked at connection time. Recheck established
// browser sessions too so lockdown, disable, and disconnect stop live audio.
const previousSessionIds = [...mediaSessions.keys()];
const sessions = await listMediaMtxItems('webrtcsessions');
const liveSessionIds = new Set(sessions.map((session) => session.id));
for (const session of sessions) {
if (!session.path?.startsWith('room-audio/')) continue;
const info = videoSessions.getSession(mediaSessions.get(session.id));
const socket = info && io.sockets.sockets.get(info.socketId);
const allowed = socket && (session.state === 'publish'
? info.sourceType === 'roomMic' && canPublish(socket, info.sourceId)
: info.sourceType === 'roomAudio' && canListen(socket));
if (!allowed) await fetch(`http://127.0.0.1:9997/v3/webrtcsessions/kick/${session.id}`, { method: 'POST', signal: AbortSignal.timeout(1500) });
}
for (const id of previousSessionIds) if (!liveSessionIds.has(id)) mediaSessions.delete(id);
} catch (error) {
logger.warn('Room audio permission refresh failed: %s', error.message);
}
// Permission API failures must not make ready streams disappear. Only path
// discovery determines the catalog used by playback and replay capture.
try {
const paths = await listMediaMtxItems('paths');
for (const path of paths) if (path.ready) ready.add(path.name);
} catch { ready = new Set(); }
finally { polling = false; }
if (generation !== currentGeneration) return;
const next = [...sources.values()].filter((source) => ready.has(`room-audio/${source.id}`)).map(({ id, name }) => ({ id, name }));
if (JSON.stringify(next) !== JSON.stringify(active)) {
active = next;
roomAudioEvents.emit('update');
}
}
io.on('connection', (socket) => {
socket.on('roomAudio:start', (_payload, cb = () => {}) => {
if (!enabled || !isLocalNetwork(getSocketIp(socket))) return cb({ error: 'Room microphone publishing requires a local visitor and enabled room audio.' });
// One owned path per socket; retries replace credentials without accumulating streams.
const id = `mic-${socket.id}`;
videoSessions.revokeWhere((info) => info.sourceType === 'roomMic' && info.sourceId === id);
sources.set(id, { id, name: 'Room microphone', socketId: socket.id });
const token = videoSessions.createSession(socket, { type: 'roomMic', id });
cb({ id, token, url: `/video/room-audio/${id}/whip` });
});
socket.on('roomAudio:release', ({ token } = {}) => {
const info = videoSessions.getSession(token);
if (info?.socketId === socket.id && info.sourceType === 'roomAudio') videoSessions.revokeSession(token);
});
socket.on('roomAudio:stop', (_payload, cb = () => {}) => {
stopSource(`mic-${socket.id}`);
cb({ success: true });
});
socket.on('disconnect', () => {
stopSource(`mic-${socket.id}`);
});
});
applyConfig(loadConfig().roomAudio);
registerConfigurationHandler('roomAudio', applyConfig);
const pollTimer = setInterval(refreshStreams, 2000);
pollTimer.unref();
process.once('exit', () => { for (const source of sources.values()) source.child?.kill('SIGTERM'); });
module.exports = { getActiveRoomAudioStreams, trackMediaSession, roomAudioEvents, getRoomAudioStreams, canListen, canPublish, hasStream };
@@ -12,6 +12,7 @@ const { managerEvents } = roverManager;
const assignmentService = require('../assignmentService'); const assignmentService = require('../assignmentService');
const { getActiveDrivers, getTurnQueues, canDrive, turnEvents } = require('../turnService'); const { getActiveDrivers, getTurnQueues, canDrive, turnEvents } = require('../turnService');
const { buildRoverTurn } = require('../turnService/display'); const { buildRoverTurn } = require('../turnService/display');
const { getRoomAudioStreams, roomAudioEvents } = require('../roomAudioService');
const { getRoomCameras, roomCameraEvents } = require('../roomCameraService'); const { getRoomCameras, roomCameraEvents } = require('../roomCameraService');
const { const {
getPublicState: getPtzCameraState, getPublicState: getPtzCameraState,
@@ -214,6 +215,7 @@ function buildSession(socket) {
status: assignmentRoverId ? assignment.status : assignment.status === 'waiting' ? 'waiting' : null, status: assignmentRoverId ? assignment.status : assignment.status === 'waiting' ? 'waiting' : null,
}, },
roomCameras: getRoomCameras(), roomCameras: getRoomCameras(),
roomAudioStreams: getRoomAudioStreams(socket),
ptzCamera: getPtzCameraState(socket), ptzCamera: getPtzCameraState(socket),
homeAssistant: getHomeAssistantState(), homeAssistant: getHomeAssistantState(),
homeAssistantActivities: getActivityState(), homeAssistantActivities: getActivityState(),
@@ -407,6 +409,8 @@ turnEvents.on('queue', (event = {}) => {
syncAll(); syncAll();
}); });
roomAudioEvents.on('update', () => syncAll());
roomCameraEvents.on('update', () => { roomCameraEvents.on('update', () => {
logger.info('Room camera change detected; syncing all clients'); logger.info('Room camera change detected; syncing all clients');
syncAll(); syncAll();
@@ -11,6 +11,7 @@ function registerVideoAuthRoute(deps) {
logAdminEvent, logAdminEvent,
extractStreamInfoFromBody, extractStreamInfoFromBody,
canAccessStream, canAccessStream,
trackRoomAudioSession,
} = deps; } = deps;
app.post('/mediamtx/auth', (req, res) => { app.post('/mediamtx/auth', (req, res) => {
@@ -36,7 +37,8 @@ function registerVideoAuthRoute(deps) {
const isLoopback = ip === '127.0.0.1' || ip === '::1'; const isLoopback = ip === '127.0.0.1' || ip === '::1';
const isRoverForwardAudioRead = action === 'read' const isRoverForwardAudioRead = action === 'read'
&& isRtspProtocol && isRtspProtocol
&& streamInfo?.id?.endsWith('-fwd'); && streamInfo?.type === 'rover'
&& streamInfo.id.endsWith('-fwd');
// Replay and snapshot workers read MediaMTX through loopback RTSP. They do // Replay and snapshot workers read MediaMTX through loopback RTSP. They do
// not represent a browser session. Rovers likewise read their dedicated // not represent a browser session. Rovers likewise read their dedicated
// -fwd speaker feed without browser credentials; every other non-loopback // -fwd speaker feed without browser credentials; every other non-loopback
@@ -50,7 +52,7 @@ function registerVideoAuthRoute(deps) {
callback for RTSP, so explicitly admit that publish protocol while leaving WHEP reads callback for RTSP, so explicitly admit that publish protocol while leaving WHEP reads
under the existing session and role checks below. under the existing session and role checks below.
*/ */
if (action === 'publish' && isRtspProtocol) { if (action === 'publish' && isRtspProtocol && (streamInfo?.type !== 'roomAudio' || isLoopback)) {
return res.status(200).end(); return res.status(200).end();
} }
@@ -62,7 +64,9 @@ function registerVideoAuthRoute(deps) {
const info = videoSessions.getSession(sessionId); const info = videoSessions.getSession(sessionId);
const streamTypeMatches = const streamTypeMatches =
info && info &&
(info.sourceType === streamInfo.type || (info.sourceType === 'roverMic' && streamInfo.type === 'rover')); (info.sourceType === streamInfo.type
|| (info.sourceType === 'roomMic' && streamInfo.type === 'roomAudio')
|| (info.sourceType === 'roverMic' && streamInfo.type === 'rover'));
if (!info || !streamTypeMatches || info.sourceId !== streamInfo.id) { if (!info || !streamTypeMatches || info.sourceId !== streamInfo.id) {
logger.warn('invalid session %s for stream %s:%s', sessionId, streamInfo.type, streamInfo.id); logger.warn('invalid session %s for stream %s:%s', sessionId, streamInfo.type, streamInfo.id);
return res.status(401).end(); return res.status(401).end();
@@ -78,6 +82,7 @@ function registerVideoAuthRoute(deps) {
return res.status(401).end(); return res.status(401).end();
} }
if (streamInfo.type === 'roomAudio' && protocol === 'webrtc') trackRoomAudioSession(body.id, sessionId);
return res.status(200).end(); return res.status(200).end();
}); });
} }
@@ -4,6 +4,7 @@
const { app } = require('../../globals/http'); const { app } = require('../../globals/http');
const io = require('../../globals/io'); const io = require('../../globals/io');
const logger = require('../../globals/logger').child('videoAuth'); const logger = require('../../globals/logger').child('videoAuth');
const roomAudioService = require('../roomAudioService');
const videoSessions = require('../videoSessions'); const videoSessions = require('../videoSessions');
const { getMode, MODES } = require('../modeManager'); const { getMode, MODES } = require('../modeManager');
const { isAdmin, isLockdownAdmin, getRole } = require('../roleService'); const { isAdmin, isLockdownAdmin, getRole } = require('../roleService');
@@ -28,6 +29,7 @@ const { canAccessStream } = createVideoAuthPolicy({
turnService, turnService,
roverManager, roverManager,
ptzCameraService, ptzCameraService,
roomAudioService,
getSocketIp, getSocketIp,
isLocalNetwork, isLocalNetwork,
io, io,
@@ -42,6 +44,7 @@ registerVideoAuthRoute({
logAdminEvent, logAdminEvent,
extractStreamInfoFromBody, extractStreamInfoFromBody,
canAccessStream, canAccessStream,
trackRoomAudioSession: roomAudioService.trackMediaSession,
}); });
module.exports = {}; module.exports = {};
@@ -17,6 +17,7 @@ function createVideoAuthPolicy(deps) {
turnService, turnService,
roverManager, roverManager,
ptzCameraService, ptzCameraService,
roomAudioService,
getSocketIp, getSocketIp,
isLocalNetwork, isLocalNetwork,
io, io,
@@ -58,6 +59,11 @@ function createVideoAuthPolicy(deps) {
} }
function canAccessStream({ socket, streamInfo, action, sourceType }) { function canAccessStream({ socket, streamInfo, action, sourceType }) {
if (streamInfo.type === 'roomAudio') {
return action === 'publish'
? sourceType === 'roomMic' && roomAudioService.canPublish(socket, streamInfo.id)
: action === 'read' && sourceType === 'roomAudio' && roomAudioService.canListen(socket) && roomAudioService.hasStream(streamInfo.id);
}
if (!canView(socket)) { if (!canView(socket)) {
return false; return false;
} }
@@ -33,6 +33,8 @@ function extractStreamInfo(path) {
return { type: 'rover', id: rawId, baseId }; return { type: 'rover', id: rawId, baseId };
} }
if (remaining.length === 2 && remaining[0] === 'room-audio') return { type: 'roomAudio', id: remaining[1] };
if (remaining.length === 2 && remaining[0] === 'room') { if (remaining.length === 2 && remaining[0] === 'room') {
return { type: 'room', id: remaining[1] || '' }; return { type: 'room', id: remaining[1] || '' };
} }
@@ -18,7 +18,9 @@ const {
function buildWhepUrlForSource(source) { function buildWhepUrlForSource(source) {
const segments = []; const segments = [];
if (source.type === 'room') { if (source.type === 'roomAudio') {
segments.push('room-audio', encodeURIComponent(source.id));
} else if (source.type === 'room') {
segments.push('room', encodeURIComponent(source.id)); segments.push('room', encodeURIComponent(source.id));
} else if (source.type === 'ptz') { } else if (source.type === 'ptz') {
/* /*
@@ -145,6 +147,9 @@ io.on('connection', (socket) => {
*/ */
throw new Error('Live video is limited to this rover queue'); throw new Error('Live video is limited to this rover queue');
} }
} else if (target.type === 'roomAudio') {
const roomAudio = require('../roomAudioService');
if (!roomAudio.canListen(socket) || !roomAudio.hasStream(target.id)) throw new Error('Room audio unavailable');
} else if (target.type === 'room') { } else if (target.type === 'room') {
throw new Error('Room cameras now use the snapshot feed'); throw new Error('Room cameras now use the snapshot feed');
} else if (target.type === 'ptz') { } else if (target.type === 'ptz') {
+3 -1
View File
@@ -4,7 +4,9 @@
"version": "0.0.0", "version": "0.0.0",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "VITE_ROVERD_URL=https://rover.otter.land vite", "predev": "node scripts/prepareMediaMtx.js",
"dev": "VITE_ROVERD_URL=https://rovertest.otter.land vite",
"prebuild": "node scripts/prepareMediaMtx.js",
"build": "vite build", "build": "vite build",
"lint": "eslint .", "lint": "eslint .",
"test": "node --test src/controls/inputs/*.test.js src/components/GamepadMappingSettings/*.test.js", "test": "node --test src/controls/inputs/*.test.js src/components/GamepadMappingSettings/*.test.js",
+45
View File
@@ -0,0 +1,45 @@
import { createHash } from 'node:crypto';
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
const version = '1.15.3'; // Matches the Dockerfile's default MediaMTX version.
const baseUrl = `https://raw.githubusercontent.com/bluenviron/mediamtx/v${version}/`;
const output = new URL('../.cache/mediamtx/publisher.js', import.meta.url);
const moduleHash = 'e903555223f196aca86d7adce5cf4437824435291bf9b47f284aa767fffabaaa';
const hash = (content) => createHash('sha256').update(content).digest('hex');
async function download(path, checksum) {
const response = await fetch(`${baseUrl}${path}`, { signal: AbortSignal.timeout(30000) });
if (!response.ok) throw new Error(`MediaMTX download failed: ${path} (${response.status})`);
const content = await response.text();
if (hash(content) !== checksum) throw new Error(`MediaMTX checksum mismatch: ${path}`);
return content;
}
let cached;
try {
cached = await readFile(output, 'utf8');
} catch (error) {
if (error.code !== 'ENOENT') throw error;
}
if (cached && hash(cached) === moduleHash) {
console.info(`Using cached MediaMTX publisher v${version}`);
} else {
const [source, license] = await Promise.all([
download('internal/servers/webrtc/publisher.js', '6b4448939b45c30507bce59acaf434e3133e6efac3bd77d7e20fb01462958509'),
download('LICENSE', 'ecae73b0a23185a35a1222edc0aae5245bb9739420bad0125b0c42e935301a80'),
]);
// Keep upstream behavior and its license; only adapt the browser-global export for Vite.
const module = `// MediaMTX v${version} publisher client, matching the Dockerfile default.\n`
+ `// Source: ${baseUrl}internal/servers/webrtc/publisher.js\n`
+ '// Only modification: export the class as an ES module for Vite.\n'
+ `/*\n${license}\n*/\n`
+ source.replace('window.MediaMTXWebRTCPublisher = MediaMTXWebRTCPublisher;', 'export default MediaMTXWebRTCPublisher;');
if (hash(module) !== moduleHash) throw new Error('Generated MediaMTX module checksum mismatch');
await mkdir(new URL('.', output), { recursive: true });
// Publish the verified module atomically so an interrupted download cannot poison the cache.
const temporary = new URL(`${output.href}.${process.pid}.tmp`);
await writeFile(temporary, module);
await rename(temporary, output);
console.info(`Downloaded and verified MediaMTX publisher v${version}`);
}
+4 -2
View File
@@ -5,6 +5,7 @@ import { useEffect, useRef, useState } from 'react';
import { useVideoRequests } from '../../hooks/useVideoRequests.js'; import { useVideoRequests } from '../../hooks/useVideoRequests.js';
import { startWhepPlayback } from '../../lib/whepPlayback.js'; import { startWhepPlayback } from '../../lib/whepPlayback.js';
import useWhepRestart from '../../hooks/useWhepRestart.js'; import useWhepRestart from '../../hooks/useWhepRestart.js';
import useServerUrl from '../../hooks/useServerUrl.js';
export const PTZ_CAMERA_ID = 'ptz-camera'; export const PTZ_CAMERA_ID = 'ptz-camera';
@@ -23,6 +24,7 @@ export default function PtzLiveVideo({
statusClassName = 'pointer-events-none absolute left-1 top-1 z-20 font-medium text-slate-100 text-[0.65rem]', statusClassName = 'pointer-events-none absolute left-1 top-1 z-20 font-medium text-slate-100 text-[0.65rem]',
fallback = null, fallback = null,
}) { }) {
const serverUrl = useServerUrl();
const videoRef = useRef(null); const videoRef = useRef(null);
const playTimerRef = useRef(null); const playTimerRef = useRef(null);
const [status, setStatus] = useState('idle'); const [status, setStatus] = useState('idle');
@@ -44,7 +46,7 @@ export default function PtzLiveVideo({
if (!enabled || shouldUseFallback || !source?.url || !videoRef.current) return undefined; if (!enabled || shouldUseFallback || !source?.url || !videoRef.current) return undefined;
const video = videoRef.current; const video = videoRef.current;
const stop = startWhepPlayback({ const stop = startWhepPlayback({
url: source.url, url: serverUrl(source.url),
token: source.token, token: source.token,
video: videoRef.current, video: videoRef.current,
startMuted, startMuted,
@@ -76,7 +78,7 @@ export default function PtzLiveVideo({
video.removeEventListener('error', handleError); video.removeEventListener('error', handleError);
stop(); stop();
}; };
}, [enabled, scheduleRestart, shouldUseFallback, source?.token, source?.url, startMuted]); }, [enabled, scheduleRestart, shouldUseFallback, source?.token, source?.url, startMuted, serverUrl]);
useEffect(() => { useEffect(() => {
const video = videoRef.current; const video = videoRef.current;
@@ -0,0 +1,71 @@
import { useCallback, useEffect, useRef } from 'react';
import { useSessionSelector } from '../../context/SessionContext.jsx';
import { useSocket } from '../../context/SocketContext.jsx';
import { useSettingsNamespace } from '../../settings/index.js';
import { AUDIO_SETTINGS_DEFAULTS } from '../../settings/namespaces.js';
import { RESTART_DELAY_MS, startWhepPlayback } from '../../lib/whepPlayback.js';
import useWhepRestart from '../../hooks/useWhepRestart.js';
import useServerUrl from '../../hooks/useServerUrl.js';
function StreamPlayer({ id, volume }) {
const socket = useSocket();
const serverUrl = useServerUrl();
const audio = useRef(null);
const { restartToken, scheduleRestart } = useWhepRestart();
useEffect(() => {
let active = true;
let stop;
let token;
socket.timeout(5000).emit('video:request', { type: 'roomAudio', id }, (error, response = {}) => {
if (!active) {
if (response.token && socket.connected) socket.emit('roomAudio:release', { token: response.token });
return;
}
token = response.token;
if (error || response.error || !response.url) { scheduleRestart(); return; }
stop = startWhepPlayback({
...response, url: serverUrl(response.url), video: audio.current, audioOnly: true,
onStatus: () => {}, onError: () => {}, scheduleRestart,
});
});
return () => {
active = false; stop?.();
if (token && socket.connected) socket.emit('roomAudio:release', { token });
};
}, [id, socket, restartToken, scheduleRestart, serverUrl]);
useEffect(() => { if (audio.current) audio.current.volume = Math.max(0, Math.min(1, volume)); }, [volume]);
return <audio ref={audio} autoPlay />;
}
export default function RoomAudioPlayer() {
const container = useRef(null);
const streams = useSessionSelector((state) => state.session?.roomAudioStreams);
const connected = useSessionSelector((state) => state.connected);
const { value } = useSettingsNamespace('audio', AUDIO_SETTINGS_DEFAULTS);
const volume = (value?.masterVolume ?? 1) * (value?.roomVolume ?? 1);
const play = useCallback(() => {
const root = container.current;
if (!root) return;
const elements = [...root.querySelectorAll('audio')].filter((audio) => audio.srcObject);
if (!elements.length) return;
// Call play during the gesture; blocked playback retries on the interval.
elements.forEach((audio) => { audio.play().catch(() => {}); });
}, []);
useEffect(() => {
if (!connected) return undefined;
const timer = setInterval(play, RESTART_DELAY_MS);
window.addEventListener('pointerdown', play);
window.addEventListener('keydown', play);
return () => {
clearInterval(timer);
window.removeEventListener('pointerdown', play);
window.removeEventListener('keydown', play);
};
}, [connected, play]);
if (!connected) return null;
return (
<div ref={container} className="contents">
{(streams || []).map((stream) => <StreamPlayer key={stream.id} id={stream.id} volume={volume} />)}
</div>
);
}
@@ -1,6 +1,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { startWhepPlayback } from '../../lib/whepPlayback.js'; import { startWhepPlayback } from '../../lib/whepPlayback.js';
import useWhepRestart from '../../hooks/useWhepRestart.js'; import useWhepRestart from '../../hooks/useWhepRestart.js';
import useServerUrl from '../../hooks/useServerUrl.js';
import { useTelemetrySelector } from '../../context/TelemetryContext.jsx'; import { useTelemetrySelector } from '../../context/TelemetryContext.jsx';
import { import {
mainBrushAudioTelemetryEqual, mainBrushAudioTelemetryEqual,
@@ -130,6 +131,7 @@ export default function RoverMediaPlayer({
pauseVideoWhenHidden = null, pauseVideoWhenHidden = null,
}) { }) {
const assignedRoverId = useSessionSelector((state) => state.session?.assignment?.roverId ?? null); const assignedRoverId = useSessionSelector((state) => state.session?.assignment?.roverId ?? null);
const serverUrl = useServerUrl();
const effectiveRoverId = roverId ?? assignedRoverId; const effectiveRoverId = roverId ?? assignedRoverId;
const mode = useSessionSelector((state) => state.session?.mode || null); const mode = useSessionSelector((state) => state.session?.mode || null);
const configuredPauseHiddenRoverVideo = useSessionSelector( const configuredPauseHiddenRoverVideo = useSessionSelector(
@@ -415,7 +417,7 @@ export default function RoverMediaPlayer({
} }
const resetMuteId = setTimeout(() => setMuted(true), 0); const resetMuteId = setTimeout(() => setMuted(true), 0);
const stop = startWhepPlayback({ const stop = startWhepPlayback({
url: resolvedSessionInfo.url, url: serverUrl(resolvedSessionInfo.url),
token: resolvedSessionInfo.token, token: resolvedSessionInfo.token,
video: videoRef.current, video: videoRef.current,
receiveAudio: !hasDedicatedAudio, receiveAudio: !hasDedicatedAudio,
@@ -446,6 +448,7 @@ export default function RoverMediaPlayer({
hasDedicatedAudio, hasDedicatedAudio,
logAudio, logAudio,
effectiveRoverId, effectiveRoverId,
serverUrl,
]); ]);
useEffect(() => { useEffect(() => {
@@ -510,7 +513,7 @@ export default function RoverMediaPlayer({
return undefined; return undefined;
} }
return startWhepPlayback({ return startWhepPlayback({
url: resolvedAudioSessionInfo.url, url: serverUrl(resolvedAudioSessionInfo.url),
token: resolvedAudioSessionInfo.token, token: resolvedAudioSessionInfo.token,
video: audioRef.current, video: audioRef.current,
audioOnly: true, audioOnly: true,
@@ -533,6 +536,7 @@ export default function RoverMediaPlayer({
audioRestartToken, audioRestartToken,
scheduleAudioRestart, scheduleAudioRestart,
logAudio, logAudio,
serverUrl,
]); ]);
useEffect(() => { useEffect(() => {
@@ -159,6 +159,7 @@ export default function SettingsPanel() {
const masterVolume = Number.isFinite(audioSettings?.masterVolume) ? audioSettings.masterVolume : AUDIO_SETTINGS_DEFAULTS.masterVolume; const masterVolume = Number.isFinite(audioSettings?.masterVolume) ? audioSettings.masterVolume : AUDIO_SETTINGS_DEFAULTS.masterVolume;
const alertVolume = Number.isFinite(audioSettings?.alertVolume) ? audioSettings.alertVolume : AUDIO_SETTINGS_DEFAULTS.alertVolume; const alertVolume = Number.isFinite(audioSettings?.alertVolume) ? audioSettings.alertVolume : AUDIO_SETTINGS_DEFAULTS.alertVolume;
const roverVolume = Number.isFinite(audioSettings?.roverVolume) ? audioSettings.roverVolume : AUDIO_SETTINGS_DEFAULTS.roverVolume; const roverVolume = Number.isFinite(audioSettings?.roverVolume) ? audioSettings.roverVolume : AUDIO_SETTINGS_DEFAULTS.roverVolume;
const roomVolume = Number.isFinite(audioSettings?.roomVolume) ? audioSettings.roomVolume : AUDIO_SETTINGS_DEFAULTS.roomVolume;
const mainBrushDuckEnabled = const mainBrushDuckEnabled =
typeof audioSettings?.mainBrushDuckEnabled === 'boolean' typeof audioSettings?.mainBrushDuckEnabled === 'boolean'
? audioSettings.mainBrushDuckEnabled ? audioSettings.mainBrushDuckEnabled
@@ -440,6 +441,11 @@ export default function SettingsPanel() {
value={roverVolume} value={roverVolume}
onChange={handleAudioRange('roverVolume')} onChange={handleAudioRange('roverVolume')}
/> />
<RangeSetting
label="Room audio"
value={roomVolume}
onChange={handleAudioRange('roomVolume')}
/>
<SettingRow> <SettingRow>
<span className="font-semibold text-white">Main brush ducking</span> <span className="font-semibold text-white">Main brush ducking</span>
<input <input
@@ -1,19 +1,16 @@
// Vip Audio Upload Card Content // Vip Audio Upload Card Content
// Purpose: Defines the Vip Audio Upload Card Content module and the local helpers/components used in this file. // Purpose: Defines the Vip Audio Upload Card Content module and the local helpers/components used in this file.
// Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit. // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
import { fieldClass } from '../constants.js'; import { fieldClass } from '../constants.js';
import { useControlSelector } from '../../../controls/index.js'; import { useControlSelector } from '../../../controls/index.js';
import ControlHint from '../../ControlHint/index.jsx'; import ControlHint from '../../ControlHint/index.jsx';
import { useSettingsNamespace } from '../../../settings/index.js'; import { useSettingsNamespace } from '../../../settings/index.js';
import { MAX_UPLOAD_BYTES, TARGET_SAMPLE_RATE, RTC_CONFIG } from './constants.js'; import { MAX_UPLOAD_BYTES, TARGET_SAMPLE_RATE } from './constants.js';
import { bytesToBase64, buildAuthHeader } from './base64.js'; import { bytesToBase64 } from './base64.js';
import { import { waitForOutboundAudioFlow } from './whipTransport.js';
waitForIceGatheringComplete, import MediaMTXWebRTCPublisher from '../../../../.cache/mediamtx/publisher.js';
waitForPeerConnected, import useServerUrl from '../../../hooks/useServerUrl.js';
configureSenderForLowLatency,
waitForOutboundAudioFlow,
} from './whipTransport.js';
import { mergeFloatChunks, encodeWavMono16 } from './audioCodec.js'; import { mergeFloatChunks, encodeWavMono16 } from './audioCodec.js';
import StatusIndicator from './StatusIndicator.jsx'; import StatusIndicator from './StatusIndicator.jsx';
import KeyPill from './KeyPill.jsx'; import KeyPill from './KeyPill.jsx';
@@ -28,6 +25,7 @@ export default function VipAudioUploadCard({
readyMicWhip, readyMicWhip,
stopMicWhip, stopMicWhip,
}) { }) {
const serverUrl = useServerUrl();
const pttActive = useControlSelector((control) => Boolean(control.state.mic?.pttActive)); const pttActive = useControlSelector((control) => Boolean(control.state.mic?.pttActive));
const { value: vipAudio, save: saveVipAudio } = useSettingsNamespace('vipAudio', { const { value: vipAudio, save: saveVipAudio } = useSettingsNamespace('vipAudio', {
openMicEnabled: false, openMicEnabled: false,
@@ -47,7 +45,8 @@ export default function VipAudioUploadCard({
const streamRef = useRef(null); const streamRef = useRef(null);
const audioTrackRef = useRef(null); const audioTrackRef = useRef(null);
const whipPcRef = useRef(null); const micAttemptRef = useRef(null);
const micEnabledRef = useRef(false);
const micActiveRef = useRef(false); const micActiveRef = useRef(false);
const activeRoverRef = useRef(''); const activeRoverRef = useRef('');
@@ -146,19 +145,12 @@ export default function VipAudioUploadCard({
const target = String(targetRoverId || activeRoverRef.current || '').trim(); const target = String(targetRoverId || activeRoverRef.current || '').trim();
micActiveRef.current = false; micActiveRef.current = false;
setMicState('idle'); setMicState('idle');
if (whipPcRef.current) { const publisher = micAttemptRef.current?.publisher;
try { micAttemptRef.current = null;
whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop()); activeRoverRef.current = '';
} catch { publisher?.close();
// noop // The upstream client stops transport retries; release its HTTP session separately.
} if (publisher?.sessionUrl) fetch(publisher.sessionUrl, { method: 'DELETE', keepalive: true }).catch(() => {});
try {
whipPcRef.current.close();
} catch {
// noop
}
}
whipPcRef.current = null;
if (streamRef.current) { if (streamRef.current) {
try { try {
streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current.getTracks().forEach((track) => track.stop());
@@ -175,14 +167,15 @@ export default function VipAudioUploadCard({
// noop // noop
} }
} }
activeRoverRef.current = '';
}, },
[stopMicWhip], [stopMicWhip],
); );
const startWhipMic = useCallback( const startWhipMic = useCallback(
async (target) => { async (target, attempt) => {
const isCurrent = () => micAttemptRef.current === attempt && activeRoverRef.current === target;
const startPayload = await startMicWhip?.(target); const startPayload = await startMicWhip?.(target);
if (!isCurrent()) return;
const whipUrl = String(startPayload?.whipUrl || '').trim(); const whipUrl = String(startPayload?.whipUrl || '').trim();
const token = String(startPayload?.token || '').trim(); const token = String(startPayload?.token || '').trim();
if (!whipUrl || !token) { if (!whipUrl || !token) {
@@ -198,12 +191,16 @@ export default function VipAudioUploadCard({
autoGainControl: false, autoGainControl: false,
}, },
}); });
if (!isCurrent()) {
stream.getTracks().forEach((track) => track.stop());
return;
}
streamRef.current = stream; streamRef.current = stream;
const track = stream.getAudioTracks()?.[0]; const track = stream.getAudioTracks()?.[0];
audioTrackRef.current = track || null; audioTrackRef.current = track || null;
if (track) { if (track) {
track.enabled = Boolean(openMicEnabled || pttActive); track.enabled = micEnabledRef.current;
} }
if (track?.applyConstraints) { if (track?.applyConstraints) {
try { try {
@@ -219,47 +216,42 @@ export default function VipAudioUploadCard({
} }
} }
const pc = new RTCPeerConnection(RTC_CONFIG); if (!isCurrent()) return;
whipPcRef.current = pc; attempt.publisher = new MediaMTXWebRTCPublisher({
pc.onconnectionstatechange = () => { url: serverUrl(whipUrl),
if (!micActiveRef.current) return; user: token,
const state = pc.connectionState; pass: token,
if (state === 'connected') { stream,
setMicState('live'); audioCodec: 'opus',
return; audioBitrate: 64,
} audioVoice: true,
if (state === 'failed' || state === 'disconnected' || state === 'closed') { onConnected: async () => {
setMicState('error'); if (!isCurrent()) return;
setMessage(`WHIP transport ${state}.`); const pc = attempt.publisher.pc;
} try {
}; // Report ready only after audio is flowing, including on transport reconnection.
stream.getAudioTracks().forEach((audioTrack) => { await waitForOutboundAudioFlow(pc);
const sender = pc.addTrack(audioTrack, stream); if (!isCurrent() || attempt.publisher.pc !== pc) return;
configureSenderForLowLatency(sender); await readyMicWhip?.(target);
}); if (!isCurrent() || attempt.publisher.pc !== pc) return;
setMicState('live');
const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false }); setMessage('');
await pc.setLocalDescription(offer); } catch (error) {
await waitForIceGatheringComplete(pc, 1800); if (!isCurrent() || attempt.publisher.pc !== pc) return;
await stopMicCapture(target);
const response = await fetch(whipUrl, { if (micAttemptRef.current) return;
method: 'POST', setMicState('error');
headers: { setMessage(error.message || 'Failed to start mic forwarding.');
'Content-Type': 'application/sdp', }
...buildAuthHeader(token), },
onError: (message) => {
if (!isCurrent()) return;
setMicState('starting');
setMessage(message);
}, },
body: pc.localDescription?.sdp || offer.sdp,
}); });
if (!response.ok) {
throw new Error(`WHIP request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
await waitForPeerConnected(pc, 10000);
await waitForOutboundAudioFlow(pc, 6000);
await readyMicWhip?.(target);
}, },
[openMicEnabled, pttActive, readyMicWhip, startMicWhip], [readyMicWhip, startMicWhip, stopMicCapture, serverUrl],
); );
const teardownClipPipeline = useCallback(async () => { const teardownClipPipeline = useCallback(async () => {
@@ -410,8 +402,6 @@ export default function VipAudioUploadCard({
}, [ensureClipPipeline, roverId]); }, [ensureClipPipeline, roverId]);
useEffect(() => { useEffect(() => {
let cancelled = false;
async function syncLiveMic() { async function syncLiveMic() {
if (clipMode) { if (clipMode) {
await stopMicCapture(activeRoverRef.current || roverId); await stopMicCapture(activeRoverRef.current || roverId);
@@ -425,34 +415,31 @@ export default function VipAudioUploadCard({
if (micActiveRef.current && activeRoverRef.current === roverId) return; if (micActiveRef.current && activeRoverRef.current === roverId) return;
if (!openMicEnabled && !pttActive) return; if (!openMicEnabled && !pttActive) return;
const attempt = {};
micAttemptRef.current = attempt;
try { try {
setMicState('starting'); setMicState('starting');
setMessage(''); setMessage('');
micActiveRef.current = true; micActiveRef.current = true;
activeRoverRef.current = roverId; activeRoverRef.current = roverId;
await startWhipMic(roverId); await startWhipMic(roverId, attempt);
if (!cancelled) {
setMicState('live');
}
} catch (err) { } catch (err) {
if (!cancelled) { if (micAttemptRef.current === attempt) {
await stopMicCapture(roverId);
if (micAttemptRef.current) return;
setMicState('error'); setMicState('error');
setMessage(err?.message || 'Failed to start mic forwarding.'); setMessage(err?.message || 'Failed to start mic forwarding.');
} }
await stopMicCapture(roverId);
} }
} }
syncLiveMic(); syncLiveMic();
return () => {
cancelled = true;
};
}, [clipMode, openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]); }, [clipMode, openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
useEffect(() => { useLayoutEffect(() => {
micEnabledRef.current = Boolean(!clipMode && (openMicEnabled || pttActive));
const track = audioTrackRef.current; const track = audioTrackRef.current;
if (!track) return; if (track) track.enabled = micEnabledRef.current;
track.enabled = Boolean(!clipMode && (openMicEnabled || pttActive));
}, [clipMode, openMicEnabled, pttActive]); }, [clipMode, openMicEnabled, pttActive]);
useEffect(() => { useEffect(() => {
@@ -10,14 +10,3 @@ export function bytesToBase64(bytes) {
} }
return btoa(binary); return btoa(binary);
} }
function encodeBase64(value) {
if (typeof btoa === 'function') return btoa(value);
return '';
}
export function buildAuthHeader(token) {
if (!token) return {};
const encoded = encodeBase64(`${token}:${token}`);
return encoded ? { Authorization: `Basic ${encoded}` } : {};
}
@@ -3,8 +3,3 @@
// Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit. // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
export const MAX_UPLOAD_BYTES = 8 * 1024 * 1024; export const MAX_UPLOAD_BYTES = 8 * 1024 * 1024;
export const TARGET_SAMPLE_RATE = 16000; export const TARGET_SAMPLE_RATE = 16000;
export const RTC_CONFIG = {
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
@@ -1,91 +1,6 @@
// whip Transport // whip Transport
// Purpose: Defines the whip Transport module and the local helpers/components used in this file. // Purpose: Defines the whip Transport module and the local helpers/components used in this file.
// Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit. // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
export function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
return new Promise((resolve) => {
if (!pc || pc.iceGatheringState === 'complete') {
resolve();
return;
}
const timer = setTimeout(() => {
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}, timeoutMs);
function onChange() {
if (pc.iceGatheringState === 'complete') {
clearTimeout(timer);
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}
}
pc.addEventListener('icegatheringstatechange', onChange);
});
}
function isPeerTransportReady(pc) {
if (!pc) return false;
const conn = pc.connectionState;
const ice = pc.iceConnectionState;
if (conn === 'connected') return true;
if (ice === 'connected' || ice === 'completed') return true;
return false;
}
export function waitForPeerConnected(pc, timeoutMs = 10000) {
return new Promise((resolve, reject) => {
if (!pc) {
reject(new Error('Peer connection missing'));
return;
}
if (isPeerTransportReady(pc)) {
resolve();
return;
}
const timer = setTimeout(() => {
cleanup();
reject(new Error('Peer connection timeout'));
}, timeoutMs);
const onState = () => {
if (isPeerTransportReady(pc)) {
cleanup();
resolve();
} else if (
pc.connectionState === 'failed' ||
pc.connectionState === 'closed' ||
pc.iceConnectionState === 'failed'
) {
cleanup();
reject(new Error(`Peer connection ${pc.connectionState || pc.iceConnectionState}`));
}
};
function cleanup() {
clearTimeout(timer);
pc.removeEventListener('connectionstatechange', onState);
pc.removeEventListener('iceconnectionstatechange', onState);
}
pc.addEventListener('connectionstatechange', onState);
pc.addEventListener('iceconnectionstatechange', onState);
});
}
export async function configureSenderForLowLatency(sender) {
if (!sender?.getParameters || !sender?.setParameters) return;
const params = sender.getParameters() || {};
const first = (params.encodings && params.encodings[0]) || {};
params.encodings = [
{
...first,
maxBitrate: 64000,
dtx: 'disabled',
},
];
try {
await sender.setParameters(params);
} catch {
// Browser support varies; keep defaults if rejected.
}
}
export function waitForOutboundAudioFlow(pc, timeoutMs = 6000) { export function waitForOutboundAudioFlow(pc, timeoutMs = 6000) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (!pc) { if (!pc) {
@@ -1,8 +1,14 @@
// Server Display App Root // Server Display App Root
// Purpose: Defines the route root for the physical-room display page. // Purpose: Defines the route root for the physical-room display page.
// Scope: Keeps this passive spectator-style surface separate from the main driver UI. // Scope: Keeps this passive spectator-style surface separate from the main driver UI.
import RoomAudioPlayer from '../../components/RoomAudioPlayer/index.jsx';
import ServerDisplayContent from './ServerDisplayContent.jsx'; import ServerDisplayContent from './ServerDisplayContent.jsx';
export default function ServerDisplayAppRoot() { export default function ServerDisplayAppRoot() {
return <ServerDisplayContent />; return (
<>
<RoomAudioPlayer />
<ServerDisplayContent />
</>
);
} }
+2 -5
View File
@@ -1,5 +1,6 @@
// Main Application Shell // Main Application Shell
// Purpose: Composes the primary rover control interface and page-level layout. Scope: Orchestrates high-level panels, overlays, and feature modules for the default route. // Purpose: Composes the primary rover control interface and page-level layout. Scope: Orchestrates high-level panels, overlays, and feature modules for the default route.
import RoomAudioPlayer from '../components/RoomAudioPlayer/index.jsx';
import { useCallback, useEffect, useState } from 'react'; import { useCallback, useEffect, useState } from 'react';
import AlertFeed from '../components/AlertFeed/index.jsx'; import AlertFeed from '../components/AlertFeed/index.jsx';
import { import {
@@ -90,11 +91,7 @@ function DriverPageContent({ layout, oldDesktop }) {
); );
return ( return (
<ControlSystemProvider> <ControlSystemProvider>
{/* <RoomAudioPlayer />
Keep document-exit protection inside the control provider so it follows
the same assigned-rover identity as every driving command. Mounting it
only on the driver page leaves spectator and utility routes unchanged.
*/}
<UndockedPageExitGuard /> <UndockedPageExitGuard />
<KeyboardInputManager /> <KeyboardInputManager />
<GamepadInputManager /> <GamepadInputManager />
+8
View File
@@ -0,0 +1,8 @@
import { useCallback } from 'react';
import { useSessionSelector } from '../context/SessionContext.jsx';
export default function useServerUrl() {
const publicUrl = useSessionSelector((state) => state.session?.interInstances?.profile?.publicUrl);
// The UI may be served by Vite while session data comes from the actual server.
return useCallback((url) => new URL(url, publicUrl || window.location.origin).href, [publicUrl]);
}
+1
View File
@@ -42,6 +42,7 @@ async function buildSocketIdentity() {
audioAdjustments: currentSettings?.audioAdjustments || {}, audioAdjustments: currentSettings?.audioAdjustments || {},
overseerEnabled: Boolean(currentSettings?.overseerPreference?.enabled), overseerEnabled: Boolean(currentSettings?.overseerPreference?.enabled),
identitySurface: getIdentitySurface(), identitySurface: getIdentitySurface(),
role: window.location.pathname === '/mic' ? 'spectator' : 'user',
operatingMode: window.location.pathname === '/ptz' ? 'ptz' : 'rover', operatingMode: window.location.pathname === '/ptz' ? 'ptz' : 'rover',
}; };
} }
+2
View File
@@ -20,6 +20,7 @@ import { SettingsProvider } from './settings/index.js'
import DeterrenceChaos from './components/DeterrenceChaos/index.jsx' import DeterrenceChaos from './components/DeterrenceChaos/index.jsx'
import AnalyticsReporter from './analytics/AnalyticsReporter.jsx' import AnalyticsReporter from './analytics/AnalyticsReporter.jsx'
import PtzAppRoot from './ptz/PtzAppRoot.jsx' import PtzAppRoot from './ptz/PtzAppRoot.jsx'
import MicApp from './mic/MicApp.jsx'
import OperatingModeRouteSync from './components/OperatingModeRouteSync/index.jsx' import OperatingModeRouteSync from './components/OperatingModeRouteSync/index.jsx'
import InitialSessionOverlay from './components/InitialSessionOverlay/index.jsx' import InitialSessionOverlay from './components/InitialSessionOverlay/index.jsx'
import AppErrorBoundary from './components/AppErrorBoundary.jsx' import AppErrorBoundary from './components/AppErrorBoundary.jsx'
@@ -65,6 +66,7 @@ createRoot(document.getElementById('root')).render(
<Route path="/spectate" element={<SpectatorApp />} /> <Route path="/spectate" element={<SpectatorApp />} />
<Route path="/mini" element={<MiniSummaryApp />} /> <Route path="/mini" element={<MiniSummaryApp />} />
<Route path="/display" element={<ServerDisplayApp />} /> <Route path="/display" element={<ServerDisplayApp />} />
<Route path="/mic" element={<MicApp />} />
<Route path="/scanner" element={<ScannerApp />} /> <Route path="/scanner" element={<ScannerApp />} />
<Route path="/setup" element={lazyRoute(<SetupApp />, 'setup')} /> <Route path="/setup" element={lazyRoute(<SetupApp />, 'setup')} />
<Route path="/admin" element={lazyRoute(<AdminApp />, 'administration')} /> <Route path="/admin" element={lazyRoute(<AdminApp />, 'administration')} />
+192
View File
@@ -0,0 +1,192 @@
import { useEffect, useLayoutEffect, useRef, useState } from 'react';
import { useSocket } from '../context/SocketContext.jsx';
import { useSessionSelector } from '../context/SessionContext.jsx';
import useUserIdentitySync from '../hooks/useUserIdentitySync.js';
import { RESTART_DELAY_MS } from '../lib/whepPlayback.js';
import { ControlSystemProvider, KeyboardInputManager, GamepadInputManager, useControlSelector } from '../controls/index.js';
import ControlHint from '../components/ControlHint/index.jsx';
import MediaMTXWebRTCPublisher from '../../.cache/mediamtx/publisher.js';
import useServerUrl from '../hooks/useServerUrl.js';
export default function MicApp() {
return (
<ControlSystemProvider>
<KeyboardInputManager />
<GamepadInputManager />
<MicPage />
</ControlSystemProvider>
);
}
function MicPage() {
const socket = useSocket();
const serverUrl = useServerUrl();
const connected = useSessionSelector((state) => state.connected);
const local = useSessionSelector((state) => state.session?.isLocalNetwork);
const enabled = useSessionSelector((state) => state.session?.features?.roomAudio);
const [status, setStatus] = useState('Connecting…');
const [pttMode, setPttMode] = useState(false);
const available = Boolean(connected && local && enabled);
const pttActive = useControlSelector((control) => Boolean(control.state.mic?.pttActive));
const [pointerHeld, setPointerHeld] = useState(false);
const [buttonKeyHeld, setButtonKeyHeld] = useState(false);
const held = available && pttMode && (pttActive || pointerHeld || buttonKeyHeld);
const transmitting = available && (!pttMode || held);
const mediaRef = useRef(null);
const transmittingRef = useRef(transmitting);
useLayoutEffect(() => {
transmittingRef.current = transmitting;
mediaRef.current?.getAudioTracks().forEach((track) => { track.enabled = transmitting; });
}, [transmitting]);
useEffect(() => {
// Pointer capture can outlive focus; release the page button when leaving the page.
function releaseButton() {
setPointerHeld(false);
setButtonKeyHeld(false);
}
function visibilityChange() {
if (document.hidden) releaseButton();
}
window.addEventListener('blur', releaseButton);
document.addEventListener('visibilitychange', visibilityChange);
return () => {
window.removeEventListener('blur', releaseButton);
document.removeEventListener('visibilitychange', visibilityChange);
};
}, []);
useUserIdentitySync();
useEffect(() => {
if (!connected || !local || !enabled) return undefined;
let disposed = false;
let attempt;
let media;
let retryTimer;
const ack = (event, payload = {}) => new Promise((resolve, reject) => {
socket.timeout(5000).emit(event, payload, (error, result = {}) => {
if (error || result.error) reject(error || new Error(result.error));
else resolve(result);
});
});
const isCurrent = (current) => !disposed && Boolean(current) && attempt === current;
function closeAttempt() {
const current = attempt;
attempt = null;
if (!current) return;
const publisher = current.publisher;
publisher?.close();
// Upstream close stops the peer and retry timer but leaves HTTP session cleanup to us.
if (publisher?.sessionUrl) fetch(publisher.sessionUrl, { method: 'DELETE', keepalive: true }).catch(() => {});
}
function retry(current, message) {
if (!isCurrent(current)) return;
closeAttempt();
clearTimeout(retryTimer);
setStatus(message);
retryTimer = setTimeout(start, RESTART_DELAY_MS);
}
async function start() {
if (disposed) return;
// Invalidate the previous attempt before closing it. Late async results
// and peer events must never change the new attempt's state or timers.
closeAttempt();
const current = {};
attempt = current;
setStatus('Starting microphone…');
try {
await ack('session:setRole', { role: 'spectator' });
if (!isCurrent(current)) return;
if (!media || media.getAudioTracks().every((track) => track.readyState === 'ended')) {
media?.getTracks().forEach((track) => { track.onended = null; track.stop(); });
if (!navigator.mediaDevices?.getUserMedia) throw new Error('Microphone access requires HTTPS or localhost');
const captured = await navigator.mediaDevices.getUserMedia({ audio: true });
if (!isCurrent(current)) { captured.getTracks().forEach((track) => track.stop()); return; }
media = captured;
mediaRef.current = media;
// New capture and reconnect attempts must respect the current PTT state immediately.
media.getAudioTracks().forEach((track) => { track.enabled = transmittingRef.current; });
media.getAudioTracks().forEach((track) => { track.onended = () => retry(attempt, 'Microphone disconnected. Retrying…'); });
}
const source = await ack('roomAudio:start');
if (!isCurrent(current)) return;
setStatus('Connecting audio…');
current.publisher = new MediaMTXWebRTCPublisher({
url: serverUrl(source.url),
user: source.token,
pass: source.token,
stream: media,
audioCodec: 'opus',
audioBitrate: 64,
audioVoice: true,
onConnected: () => {
if (isCurrent(current)) setStatus('Microphone live');
},
// The publisher owns transport retries; restarting it here would create competing loops.
onError: (message) => {
if (isCurrent(current)) setStatus(message);
},
});
} catch (error) { retry(current, `${error.message}. Retrying…`); }
}
start();
return () => {
disposed = true;
clearTimeout(retryTimer);
closeAttempt();
media?.getTracks().forEach((track) => { track.onended = null; track.stop(); });
mediaRef.current = null;
if (socket.connected) socket.emit('roomAudio:stop', {});
};
}, [socket, connected, local, enabled, serverUrl]);
const message = !connected ? 'Connecting…' : local === false ? 'This page is available to local visitors only.' : !enabled ? 'Room audio is disabled.' : status;
return (
<main className="min-h-screen bg-neutral-950 p-4 text-slate-200">
<div className="panel mx-auto max-w-md p-4">
<h1 className="text-lg font-semibold">Room microphone</h1>
<p className="mt-2 text-sm" role="status">{message}</p>
<div className="mt-4 grid grid-cols-2 gap-2" role="group" aria-label="Microphone mode">
{[false, true].map((ptt) => (
<button key={String(ptt)} type="button" aria-pressed={pttMode === ptt}
className={`button-dark w-full ${pttMode === ptt ? 'bg-emerald-500 text-white hover:bg-emerald-500' : ''}`}
onClick={() => {
setPointerHeld(false);
setButtonKeyHeld(false);
setPttMode(ptt);
}}>
{ptt ? 'Push to talk' : 'Open mic'}
</button>
))}
</div>
{pttMode && (
<>
<button type="button" disabled={!available} aria-pressed={held}
className={`button-dark mt-4 min-h-40 w-full touch-none select-none text-2xl font-semibold ${held ? 'bg-emerald-500 text-white hover:bg-emerald-500' : ''}`}
onPointerDown={(event) => {
if (event.button !== 0 || !event.isPrimary) return;
event.currentTarget.setPointerCapture(event.pointerId);
setPointerHeld(true);
}}
onPointerUp={() => setPointerHeld(false)}
onPointerCancel={() => setPointerHeld(false)}
onLostPointerCapture={() => setPointerHeld(false)}
onContextMenu={(event) => event.preventDefault()}
onKeyDown={(event) => {
if (event.key !== ' ' && event.key !== 'Enter') return;
event.preventDefault();
setButtonKeyHeld(true);
}}
onKeyUp={(event) => {
if (event.key !== ' ' && event.key !== 'Enter') return;
event.preventDefault();
setButtonKeyHeld(false);
}}
onBlur={() => setButtonKeyHeld(false)}>
{held ? 'Talking…' : 'Hold to talk'}
</button>
<p className="mt-2 text-center text-sm text-slate-400">Hold the button or <ControlHint actionId="micPtt" /> to talk.</p>
</>
)}
</div>
</main>
);
}
@@ -1,6 +1,7 @@
// Mini Summary App Root // Mini Summary App Root
// Purpose: Defines the Mini Summary App Root module and the local helpers/components used in this file. // Purpose: Defines the Mini Summary App Root module and the local helpers/components used in this file.
// Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit. // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
import RoomAudioPlayer from '../../components/RoomAudioPlayer/index.jsx';
import { useEffect } from 'react'; import { useEffect } from 'react';
import AlertFeed from '../../components/AlertFeed/index.jsx'; import AlertFeed from '../../components/AlertFeed/index.jsx';
import MiniSummaryContent from './MiniSummaryContent.jsx'; import MiniSummaryContent from './MiniSummaryContent.jsx';
@@ -19,6 +20,7 @@ export default function MiniSummaryAppRoot() {
return ( return (
<> <>
<RoomAudioPlayer />
<MiniSummaryContent /> <MiniSummaryContent />
<AlertFeed scale={3} opacity={1}/> <AlertFeed scale={3} opacity={1}/>
</> </>
+2
View File
@@ -2,6 +2,7 @@
// Purpose: Mounts the PTZ controller as a real page with the same shared input // Purpose: Mounts the PTZ controller as a real page with the same shared input
// and identity systems used by the driver page. // and identity systems used by the driver page.
// Scope: Mounts input/identity providers; the page owns session entry and layouts. // Scope: Mounts input/identity providers; the page owns session entry and layouts.
import RoomAudioPlayer from '../components/RoomAudioPlayer/index.jsx';
import AlertFeed from '../components/AlertFeed/index.jsx'; import AlertFeed from '../components/AlertFeed/index.jsx';
import SocketConnectionPill from '../components/SocketConnectionPill/index.jsx'; import SocketConnectionPill from '../components/SocketConnectionPill/index.jsx';
import { LayoutProvider } from '../layouts/LayoutContext.jsx'; import { LayoutProvider } from '../layouts/LayoutContext.jsx';
@@ -25,6 +26,7 @@ function PtzRouteContent() {
return ( return (
<ControlSystemProvider> <ControlSystemProvider>
<RoomAudioPlayer />
<KeyboardInputManager /> <KeyboardInputManager />
<GamepadInputManager /> <GamepadInputManager />
<LayoutProvider layout={layout}> <LayoutProvider layout={layout}>
+1
View File
@@ -176,6 +176,7 @@ export const AUDIO_SETTINGS_DEFAULTS = {
masterVolume: 1, masterVolume: 1,
alertVolume: 0.5, alertVolume: 0.5,
roverVolume: 1, roverVolume: 1,
roomVolume: 1,
mainBrushDuckEnabled: true, mainBrushDuckEnabled: true,
mainBrushDuckAmount: 0.75, mainBrushDuckAmount: 0.75,
}; };
@@ -1,8 +1,14 @@
// Spectator App Root // Spectator App Root
// Purpose: Defines the Spectator App Root module and the local helpers/components used in this file. // Purpose: Defines the Spectator App Root module and the local helpers/components used in this file.
// Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit. // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
import RoomAudioPlayer from '../../components/RoomAudioPlayer/index.jsx';
import SpectatorContent from './SpectatorContent.jsx'; import SpectatorContent from './SpectatorContent.jsx';
export default function SpectatorAppRoot() { export default function SpectatorAppRoot() {
return <SpectatorContent />; return (
<>
<RoomAudioPlayer />
<SpectatorContent />
</>
);
} }