legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 01:21:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

whipwhep

Browse Source
This commit is contained in:
legop3
2026-03-20 23:35:34 -04:00
parent cb97e73987
commit e3b428f9ab
14 changed files with 659 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plans/private rover outline.md
+13 -12
View File
@@ -1,29 +1,30 @@
# private rovers # private rovers
## basic concept: ## basic concept:
private rovers will be mostly just for lockdown admins to drive and use, but they can be temporarily unlocked manually by lockdown admins for use by verified users. private rovers will be mostly just for lockdown admins to drive and use, but they can be temporarily unlocked manually by lockdown admins for use by verified users.
This means that locking / unlocking will act a little different than standard rovers.
- cannot be spectated by spectators, ever - cannot be spectated by spectators, unless they are unlocked
- cannot be replayed, ever - cannot be replayed, unless they are unlocked
- private status is defined in the roverd config - private status is defined in the roverd config
- needs to never leak through access to anyone while locked - needs to never leak through access to anyone while locked
- unlocking a private rover is a big deal for verified users - unlocking a private rover is a big deal for verified users (opening up a rover in the main living space for a special event)
## locking / unlocking: ## locking / unlocking:
- private rovers start locked - private rovers start locked
- when locked, only lockdown admins can drive them - when locked, only lockdown admins can drive them
- when unlocked, only verified users can drive them - when unlocked, only verified users (and lockdown admins of course) can drive them
- if left unlocked with no one online for 1 hour, the server will automatically lock them - if left unlocked with no one online for 30 mins, the server will automatically lock them
## cliff rules / speed limit / overcurrent limit ## cliff rules / speed limit / overcurrent limit
### private rovers will be in a sensitive area, so their physical capabilities will be limited by the server ### private rovers will be in a sensitive area, their physical capabilities will be optionally limited by the server, controllable by lockdown admins.
- if the cliff sensors get triggered, stop the rover and back it up - optional toggleable limits:
- speed limit is already kind of a thing but has never been tested, need to make sure it works all the way through the control pipeline - speed limit
- hard overcurrent limits done server-side. completely seperate from the current client only ones. - hard overcurrent limiting (stop motor for a bit the instant it overcurrents for maybe 0.3s)
- almost zero tolerance for wheel and side brush overcurrents - hard bump limits, stop and back up slightly on physical bumps of a certain short duration
- come up with a way to do this without making it feel too punishing. overcurrents often happen by accident - cliff drops. back up and pause when any cliff sensor triggers, use their binary outputs for this as they are tuned well from factory.
- ignore the main brush, private rovers wont have one so it may read wrong
## UI specifics ## UI specifics
- private rovers don't show in the spectator pages unless they are unlocked
- private rovers don't show in the list for normal users unless they are unlocked - private rovers don't show in the list for normal users unless they are unlocked
- they will only show for lockdown admins - they will only show for lockdown admins
- when unlocked, they show for everyone - when unlocked, they show for everyone
server/public/assets/index-DcrX1ADu.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-DnqjOvSX.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-DnqjOvSX.js"></script> <script type="module" crossorigin src="/assets/index-DcrX1ADu.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-Dx4QsRNa.css"> <link rel="stylesheet" crossorigin href="/assets/index-Dx4QsRNa.css">
</head> </head>
<body> <body>
server/src/services/audioForwardService.js
+131 -17
View File
@@ -8,10 +8,12 @@ const { loadConfig } = require('../helpers/configLoader');
const roverManager = require('./roverManager'); const roverManager = require('./roverManager');
const turnService = require('./turnService'); const turnService = require('./turnService');
const { isVerified } = require('./verificationService'); const { isVerified } = require('./verificationService');
const videoSessions = require('./videoSessions');
const audioForwardEvents = new EventEmitter(); const audioForwardEvents = new EventEmitter();
const config = loadConfig(); const config = loadConfig();
const audioForwardConfig = config.audioForward || {}; const audioForwardConfig = config.audioForward || {};
const mediaConfig = config.media || {};
const serviceEnabled = audioForwardConfig.enabled !== false; const serviceEnabled = audioForwardConfig.enabled !== false;
const ffmpegBin = audioForwardConfig.ffmpegBin || 'ffmpeg'; const ffmpegBin = audioForwardConfig.ffmpegBin || 'ffmpeg';
const streamSuffix = const streamSuffix =
@@ -26,6 +28,7 @@ const maxUploadBytes = Number.isFinite(audioForwardConfig.maxUploadBytes)
const states = new Map(); // roverId -> { state, source, error, startedAt, updatedAt } const states = new Map(); // roverId -> { state, source, error, startedAt, updatedAt }
const workers = new Map(); // roverId -> worker const workers = new Map(); // roverId -> worker
const whipOwners = new Map(); // roverId -> socketId
function publishStateChange(roverId) { function publishStateChange(roverId) {
audioForwardEvents.emit('change', { roverId, state: states.get(roverId) || null }); audioForwardEvents.emit('change', { roverId, state: states.get(roverId) || null });
@@ -126,6 +129,29 @@ function resolveForwardUrl(roverId) {
)},m=publish&latency=10&mode=caller&transtype=live&pkt_size=1316`; )},m=publish&latency=10&mode=caller&transtype=live&pkt_size=1316`;
} }
function resolveForwardPathId(roverId) {
return `${roverId}${streamSuffix}`;
}
function getMediaPrefix() {
const base = mediaConfig.whepBaseUrl;
if (!base) return '';
try {
const parsed = new URL(base);
return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, '');
} catch {
return String(base).replace(/\/+$/, '');
}
}
function buildWhipUrl(pathId) {
const prefix = getMediaPrefix();
if (!prefix) {
throw new Error('Server media base URL missing');
}
return `${prefix}/${encodeURIComponent(pathId)}/whip`;
}
function spawnFfmpeg(roverId, tag, args, options = {}) { function spawnFfmpeg(roverId, tag, args, options = {}) {
const proc = spawn(ffmpegBin, args, { const proc = spawn(ffmpegBin, args, {
stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'], stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'],
@@ -399,6 +425,12 @@ function ensureWorker(roverId) {
} }
function stopWorker(roverId) { function stopWorker(roverId) {
const whipOwner = whipOwners.get(roverId);
if (whipOwner) {
whipOwners.delete(roverId);
revokeWhipSessionForRover(roverId, whipOwner);
}
const worker = workers.get(roverId); const worker = workers.get(roverId);
if (!worker) return; if (!worker) return;
@@ -426,16 +458,12 @@ function writeUploadFile(roverId, payload = {}) {
const { name, mime, dataBase64 } = payload || {}; const { name, mime, dataBase64 } = payload || {};
const ext = extFromUpload(name, mime); const ext = extFromUpload(name, mime);
const encoded = typeof dataBase64 === 'string' ? dataBase64.trim() : ''; const encoded = typeof dataBase64 === 'string' ? dataBase64.trim() : '';
if (!encoded) { if (!encoded) throw new Error('Upload payload missing');
throw new Error('Upload payload missing');
}
const bytes = Buffer.from(encoded, 'base64'); const bytes = Buffer.from(encoded, 'base64');
if (!bytes.length) { if (!bytes.length) throw new Error('Upload decode failed');
throw new Error('Upload decode failed'); if (bytes.length > maxUploadBytes) throw new Error(`Upload too large (max ${maxUploadBytes} bytes)`);
}
if (bytes.length > maxUploadBytes) {
throw new Error(`Upload too large (max ${maxUploadBytes} bytes)`);
}
ensureRuntimeDir(); ensureRuntimeDir();
const stem = sanitizeFileStem(name || `upload-${Date.now()}`); const stem = sanitizeFileStem(name || `upload-${Date.now()}`);
const filePath = path.join(uploadsDir, `${sanitizeRoverId(roverId)}-${Date.now()}-${stem}${ext}`); const filePath = path.join(uploadsDir, `${sanitizeRoverId(roverId)}-${Date.now()}-${stem}${ext}`);
@@ -444,18 +472,52 @@ function writeUploadFile(roverId, payload = {}) {
} }
function playUploadedAudio(roverId, payload = {}, ownerSocketId = null) { function playUploadedAudio(roverId, payload = {}, ownerSocketId = null) {
stopWhipForRover(roverId, 'upload_override');
ensureWorker(roverId); ensureWorker(roverId);
const uploadPath = writeUploadFile(roverId, payload); const uploadPath = writeUploadFile(roverId, payload);
startUploadWriter(roverId, uploadPath, ownerSocketId); startUploadWriter(roverId, uploadPath, ownerSocketId);
} }
function stopPlayback(roverId) { function stopPlayback(roverId) {
stopWhipForRover(roverId, 'stop_playback');
ensureWorker(roverId); ensureWorker(roverId);
startSilenceWriter(roverId); startSilenceWriter(roverId);
} }
function stopOwnedUploadIfUnauthorized(roverId, ownerSocketId, reason = 'driver_change') { function revokeWhipSessionForRover(roverId, ownerSocketId) {
if (!roverId || !ownerSocketId) return; if (!roverId || !ownerSocketId) return;
const pathId = resolveForwardPathId(roverId);
videoSessions.revokeWhere(
(info) => info?.socketId === ownerSocketId && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
}
function stopWhipForRover(roverId, reason = 'unknown') {
const ownerSocketId = whipOwners.get(roverId);
if (!ownerSocketId) return;
whipOwners.delete(roverId);
revokeWhipSessionForRover(roverId, ownerSocketId);
logger.info('Stopping WHIP mic session', { roverId, ownerSocketId, reason });
try {
ensureWorker(roverId);
startSilenceWriter(roverId);
} catch (err) {
setState(roverId, { state: 'error', source: 'mic-whip', error: err?.message || String(err), startedAt: null });
}
}
function stopOwnedAudioIfUnauthorized(roverId, ownerSocketId, reason = 'driver_change') {
if (!roverId || !ownerSocketId) return;
if (whipOwners.get(roverId) === ownerSocketId) {
const ownerSocket = io.sockets.sockets.get(ownerSocketId);
const ownerIsDriver = ownerSocket ? roverManager.isDriver(roverId, ownerSocket) : false;
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
if (!ownerIsDriver || !ownerCanDrive) {
stopWhipForRover(roverId, reason);
}
}
const worker = workers.get(roverId); const worker = workers.get(roverId);
if (!worker || worker.contentKind !== 'upload' || worker.activeOwnerSocketId !== ownerSocketId) return; if (!worker || worker.contentKind !== 'upload' || worker.activeOwnerSocketId !== ownerSocketId) return;
@@ -464,11 +526,7 @@ function stopOwnedUploadIfUnauthorized(roverId, ownerSocketId, reason = 'driver_
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false; const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
if (ownerIsDriver && ownerCanDrive) return; if (ownerIsDriver && ownerCanDrive) return;
logger.info('Stopping upload audio due to ownership/driver change', { logger.info('Stopping upload audio due to ownership/driver change', { roverId, ownerSocketId, reason });
roverId,
ownerSocketId,
reason,
});
startSilenceWriter(roverId); startSilenceWriter(roverId);
} }
@@ -479,6 +537,9 @@ roverManager.managerEvents.on('rover', ({ roverId, action } = {}) => {
return; return;
} }
if (action === 'upsert' && serviceEnabled) { if (action === 'upsert' && serviceEnabled) {
if (whipOwners.has(roverId)) {
return;
}
try { try {
ensureWorker(roverId); ensureWorker(roverId);
} catch (err) { } catch (err) {
@@ -490,15 +551,19 @@ roverManager.managerEvents.on('rover', ({ roverId, action } = {}) => {
roverManager.managerEvents.on('driver', ({ socketId, roverId, action } = {}) => { roverManager.managerEvents.on('driver', ({ socketId, roverId, action } = {}) => {
if (!socketId || !roverId) return; if (!socketId || !roverId) return;
if (action === 'remove' || action === 'add') { if (action === 'remove' || action === 'add') {
stopOwnedUploadIfUnauthorized(roverId, socketId, action); stopOwnedAudioIfUnauthorized(roverId, socketId, action);
} }
}); });
turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => { turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => {
if (!roverId) return; if (!roverId) return;
const whipOwner = whipOwners.get(roverId);
if (whipOwner) {
stopOwnedAudioIfUnauthorized(roverId, whipOwner, 'turn_change');
}
const worker = workers.get(roverId); const worker = workers.get(roverId);
if (!worker || worker.contentKind !== 'upload') return; if (!worker || worker.contentKind !== 'upload') return;
stopOwnedUploadIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change'); stopOwnedAudioIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
}); });
io.on('connection', (socket) => { io.on('connection', (socket) => {
@@ -528,12 +593,61 @@ io.on('connection', (socket) => {
} }
}); });
socket.on('audio:micWhipStart', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
stopWorker(normalized);
whipOwners.set(normalized, socket.id);
const pathId = resolveForwardPathId(normalized);
revokeWhipSessionForRover(normalized, socket.id);
const token = videoSessions.createSession(socket, { type: 'roverMic', id: pathId });
const whipUrl = buildWhipUrl(pathId);
setState(normalized, { state: 'starting', source: 'mic-whip', error: null, startedAt: Date.now() });
cb({ success: true, roverId: normalized, pathId, token, whipUrl });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('audio:micWhipReady', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
if (whipOwners.get(normalized) !== socket.id) {
throw new Error('WHIP session not owned by this client');
}
setState(normalized, { state: 'playing', source: 'mic-whip', error: null, startedAt: Date.now() });
cb({ success: true, roverId: normalized });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('audio:micWhipStop', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
if (whipOwners.get(normalized) && whipOwners.get(normalized) !== socket.id) {
throw new Error('Mic forwarding is owned by another session');
}
stopWhipForRover(normalized, 'client_stop');
cb({ success: true, roverId: normalized });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('disconnect', () => { socket.on('disconnect', () => {
workers.forEach((worker, roverId) => { workers.forEach((worker, roverId) => {
if (!worker || worker.contentKind !== 'upload' || worker.activeOwnerSocketId !== socket.id) return; if (!worker || worker.contentKind !== 'upload' || worker.activeOwnerSocketId !== socket.id) return;
logger.info('Stopping owned upload audio due to socket disconnect', { roverId, socketId: socket.id }); logger.info('Stopping owned upload audio due to socket disconnect', { roverId, socketId: socket.id });
startSilenceWriter(roverId); startSilenceWriter(roverId);
}); });
for (const [roverId, ownerSocketId] of whipOwners.entries()) {
if (ownerSocketId !== socket.id) continue;
stopWhipForRover(roverId, 'socket_disconnect');
}
}); });
}); });
server/src/services/videoAuthService.js
+19 -1
View File
@@ -4,6 +4,8 @@ const logger = require('../globals/logger').child('videoAuth');
const videoSessions = require('./videoSessions'); const videoSessions = require('./videoSessions');
const { getMode, MODES } = require('./modeManager'); const { getMode, MODES } = require('./modeManager');
const { isAdmin, isLockdownAdmin, getRole } = require('./roleService'); const { isAdmin, isLockdownAdmin, getRole } = require('./roleService');
const { isVerified } = require('./verificationService');
const turnService = require('./turnService');
const roverManager = require('./roverManager'); const roverManager = require('./roverManager');
const { loadConfig } = require('../helpers/configLoader'); const { loadConfig } = require('../helpers/configLoader');
const { getRequestIp, getSocketIp, isLocalNetwork } = require('../helpers/ipResolver'); const { getRequestIp, getSocketIp, isLocalNetwork } = require('../helpers/ipResolver');
@@ -145,7 +147,9 @@ app.post('/mediamtx/auth', (req, res) => {
} }
const info = videoSessions.getSession(sessionId); const info = videoSessions.getSession(sessionId);
const streamTypeMatches = info && info.sourceType === streamInfo.type; const streamTypeMatches =
info &&
(info.sourceType === streamInfo.type || (info.sourceType === 'roverMic' && streamInfo.type === 'rover'));
if (!info || !streamTypeMatches || info.sourceId !== streamInfo.id) { if (!info || !streamTypeMatches || info.sourceId !== streamInfo.id) {
logger.warn('invalid session %s for stream %s:%s', sessionId, streamInfo.type, streamInfo.id); logger.warn('invalid session %s for stream %s:%s', sessionId, streamInfo.type, streamInfo.id);
return res.status(401).end(); return res.status(401).end();
@@ -158,6 +162,20 @@ app.post('/mediamtx/auth', (req, res) => {
if (!canView(socket)) { if (!canView(socket)) {
return res.status(401).end(); return res.status(401).end();
} }
if (info.sourceType === 'roverMic' && action === 'publish') {
const roverId = streamInfo.baseId || streamInfo.id;
if (!isVerified(socket)) {
return res.status(401).end();
}
if (!roverManager.isDriver(roverId, socket)) {
return res.status(401).end();
}
if (!turnService.canDrive(roverId, socket)) {
return res.status(401).end();
}
return res.status(200).end();
}
const role = getRole(socket); const role = getRole(socket);
const isAudio = streamInfo.id?.endsWith('-audio'); const isAudio = streamInfo.id?.endsWith('-audio');
if (role === 'spectator' && !isAdmin(socket) && !isAudio) { if (role === 'spectator' && !isAdmin(socket) && !isAudio) {
webui/src/components/KeymapSettings.jsx
+1
View File
@@ -24,6 +24,7 @@ const KEY_ACTIONS = [
{ id: 'cameraDown', label: 'Camera Down', group: 'Camera' }, { id: 'cameraDown', label: 'Camera Down', group: 'Camera' },
{ id: 'nightVisionToggle', label: 'Toggle Night Vision', group: 'Camera' }, { id: 'nightVisionToggle', label: 'Toggle Night Vision', group: 'Camera' },
{ id: 'hornHonk', label: 'Horn (Hold)', group: 'Audio' }, { id: 'hornHonk', label: 'Horn (Hold)', group: 'Audio' },
{ id: 'micPtt', label: 'Mic Push To Talk', group: 'Audio' },
{ id: 'driveMacro', label: 'Drive Macro', group: 'Macros' }, { id: 'driveMacro', label: 'Drive Macro', group: 'Macros' },
{ id: 'dockMacro', label: 'Dock Macro', group: 'Macros' }, { id: 'dockMacro', label: 'Dock Macro', group: 'Macros' },
{ id: 'chatFocus', label: 'Toggle Chat', group: 'Chat' }, { id: 'chatFocus', label: 'Toggle Chat', group: 'Chat' },
webui/src/components/VipPanel.jsx
+13 -1
View File
@@ -7,7 +7,16 @@ import VipVerificationCard from './vip/VipVerificationCard.jsx';
import VipIdentityCard from './vip/VipIdentityCard.jsx'; import VipIdentityCard from './vip/VipIdentityCard.jsx';
export default function VipPanel() { export default function VipPanel() {
const { session, identifySession, requestVerification, playUploadedAudio, stopUploadedAudio } = useSession(); const {
session,
identifySession,
requestVerification,
playUploadedAudio,
stopUploadedAudio,
startMicWhip,
readyMicWhip,
stopMicWhip,
} = useSession();
const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' }); const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' });
const { value: profile } = useSettingsNamespace('profile', { nickname: '' }); const { value: profile } = useSettingsNamespace('profile', { nickname: '' });
@@ -39,6 +48,9 @@ export default function VipPanel() {
audioForwardByRover={session?.audioForward || {}} audioForwardByRover={session?.audioForward || {}}
playUploadedAudio={playUploadedAudio} playUploadedAudio={playUploadedAudio}
stopUploadedAudio={stopUploadedAudio} stopUploadedAudio={stopUploadedAudio}
startMicWhip={startMicWhip}
readyMicWhip={readyMicWhip}
stopMicWhip={stopMicWhip}
/> />
) : ( ) : (
<VipVerificationCard <VipVerificationCard
webui/src/components/vip/VipAudioUploadCard.jsx
+316 -1
View File
@@ -1,7 +1,14 @@
import { useMemo, useState } from 'react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { fieldClass, flowWrapClass, innerFlowClass } from './constants.js'; import { fieldClass, flowWrapClass, innerFlowClass } from './constants.js';
import { useControlSystem } from '../../controls/index.js';
const MAX_UPLOAD_BYTES = 8 * 1024 * 1024; const MAX_UPLOAD_BYTES = 8 * 1024 * 1024;
const TARGET_SAMPLE_RATE = 16000;
const RTC_CONFIG = {
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
function bytesToBase64(bytes) { function bytesToBase64(bytes) {
let binary = ''; let binary = '';
@@ -13,16 +20,161 @@ function bytesToBase64(bytes) {
return btoa(binary); return btoa(binary);
} }
function encodeBase64(value) {
if (typeof btoa === 'function') return btoa(value);
return '';
}
function buildAuthHeader(token) {
if (!token) return {};
const encoded = encodeBase64(`${token}:${token}`);
return encoded ? { Authorization: `Basic ${encoded}` } : {};
}
function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
return new Promise((resolve) => {
if (!pc || pc.iceGatheringState === 'complete') {
resolve();
return;
}
const timer = setTimeout(() => {
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}, timeoutMs);
function onChange() {
if (pc.iceGatheringState === 'complete') {
clearTimeout(timer);
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}
}
pc.addEventListener('icegatheringstatechange', onChange);
});
}
function isPeerTransportReady(pc) {
if (!pc) return false;
const conn = pc.connectionState;
const ice = pc.iceConnectionState;
if (conn === 'connected') return true;
if (ice === 'connected' || ice === 'completed') return true;
return false;
}
function waitForPeerConnected(pc, timeoutMs = 10000) {
return new Promise((resolve, reject) => {
if (!pc) {
reject(new Error('Peer connection missing'));
return;
}
if (isPeerTransportReady(pc)) {
resolve();
return;
}
const timer = setTimeout(() => {
cleanup();
reject(new Error('Peer connection timeout'));
}, timeoutMs);
const onState = () => {
if (isPeerTransportReady(pc)) {
cleanup();
resolve();
} else if (
pc.connectionState === 'failed' ||
pc.connectionState === 'closed' ||
pc.iceConnectionState === 'failed'
) {
cleanup();
reject(new Error(`Peer connection ${pc.connectionState || pc.iceConnectionState}`));
}
};
function cleanup() {
clearTimeout(timer);
pc.removeEventListener('connectionstatechange', onState);
pc.removeEventListener('iceconnectionstatechange', onState);
}
pc.addEventListener('connectionstatechange', onState);
pc.addEventListener('iceconnectionstatechange', onState);
});
}
async function configureSenderForLowLatency(sender) {
if (!sender?.getParameters || !sender?.setParameters) return;
const params = sender.getParameters() || {};
const first = (params.encodings && params.encodings[0]) || {};
params.encodings = [
{
...first,
maxBitrate: 64000,
dtx: 'disabled',
},
];
try {
await sender.setParameters(params);
} catch {
// Browser support varies; keep defaults if rejected.
}
}
function waitForOutboundAudioFlow(pc, timeoutMs = 6000) {
return new Promise((resolve, reject) => {
if (!pc) {
reject(new Error('Peer connection missing'));
return;
}
const start = Date.now();
let baseline = -1;
const timer = setInterval(async () => {
if (Date.now() - start > timeoutMs) {
clearInterval(timer);
reject(new Error('WHIP connected but no outbound audio flow'));
return;
}
try {
const senders = pc.getSenders().filter((s) => s.track?.kind === 'audio');
for (const sender of senders) {
const stats = await sender.getStats();
for (const report of stats.values()) {
if (report.type !== 'outbound-rtp' || report.kind !== 'audio') continue;
const sent = Number(report.bytesSent || 0);
const packets = Number(report.packetsSent || 0);
if (baseline < 0) {
baseline = sent;
} else if (sent > baseline + 200 || packets > 5) {
clearInterval(timer);
resolve();
return;
}
}
}
} catch {
// Keep polling until timeout.
}
}, 250);
});
}
export default function VipAudioUploadCard({ export default function VipAudioUploadCard({
ownRoverId = '', ownRoverId = '',
audioForwardByRover = {}, audioForwardByRover = {},
playUploadedAudio, playUploadedAudio,
stopUploadedAudio, stopUploadedAudio,
startMicWhip,
readyMicWhip,
stopMicWhip,
}) { }) {
const { state: controlState } = useControlSystem();
const roverId = String(ownRoverId || '').trim(); const roverId = String(ownRoverId || '').trim();
const [selectedUpload, setSelectedUpload] = useState(null); const [selectedUpload, setSelectedUpload] = useState(null);
const [working, setWorking] = useState(false); const [working, setWorking] = useState(false);
const [openMicEnabled, setOpenMicEnabled] = useState(false);
const [micState, setMicState] = useState('idle');
const [message, setMessage] = useState(''); const [message, setMessage] = useState('');
const streamRef = useRef(null);
const whipPcRef = useRef(null);
const micActiveRef = useRef(false);
const activeRoverRef = useRef('');
const pttActive = Boolean(controlState?.mic?.pttActive);
const selectedForwardState = useMemo( const selectedForwardState = useMemo(
() => (roverId ? audioForwardByRover?.[roverId] || null : null), () => (roverId ? audioForwardByRover?.[roverId] || null : null),
@@ -80,6 +232,152 @@ export default function VipAudioUploadCard({
} }
}; };
const stopMicCapture = useCallback(
async (targetRoverId) => {
const target = String(targetRoverId || activeRoverRef.current || '').trim();
micActiveRef.current = false;
setMicState('idle');
if (whipPcRef.current) {
try {
whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop());
} catch {
// noop
}
try {
whipPcRef.current.close();
} catch {
// noop
}
}
whipPcRef.current = null;
if (streamRef.current) {
try {
streamRef.current.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
}
streamRef.current = null;
if (target) {
try {
await stopMicWhip?.(target);
} catch {
// noop
}
}
activeRoverRef.current = '';
},
[stopMicWhip],
);
const startWhipMic = useCallback(
async (target) => {
const startPayload = await startMicWhip?.(target);
const whipUrl = String(startPayload?.whipUrl || '').trim();
const token = String(startPayload?.token || '').trim();
if (!whipUrl || !token) {
throw new Error('WHIP endpoint unavailable');
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
},
});
streamRef.current = stream;
const track = stream.getAudioTracks()?.[0];
if (track?.applyConstraints) {
try {
await track.applyConstraints({
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
});
} catch {
// noop
}
}
const pc = new RTCPeerConnection(RTC_CONFIG);
whipPcRef.current = pc;
stream.getAudioTracks().forEach((audioTrack) => {
const sender = pc.addTrack(audioTrack, stream);
configureSenderForLowLatency(sender);
});
const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false });
await pc.setLocalDescription(offer);
await waitForIceGatheringComplete(pc, 1800);
const response = await fetch(whipUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/sdp',
...buildAuthHeader(token),
},
body: pc.localDescription?.sdp || offer.sdp,
});
if (!response.ok) {
throw new Error(`WHIP request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
await waitForPeerConnected(pc, 10000);
await waitForOutboundAudioFlow(pc, 6000);
await readyMicWhip?.(target);
},
[readyMicWhip, startMicWhip],
);
useEffect(() => {
const desiredActive = Boolean(openMicEnabled || pttActive);
let cancelled = false;
async function syncMicState() {
if (!roverId || !desiredActive) {
await stopMicCapture(roverId);
return;
}
if (micActiveRef.current && activeRoverRef.current === roverId) return;
try {
setMicState('starting');
setMessage('');
micActiveRef.current = true;
activeRoverRef.current = roverId;
await startWhipMic(roverId);
if (!cancelled) {
setMicState('live');
}
} catch (err) {
if (!cancelled) {
setMicState('error');
setMessage(err?.message || 'Failed to start mic forwarding.');
}
await stopMicCapture(roverId);
}
}
syncMicState();
return () => {
cancelled = true;
};
}, [openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
useEffect(
() => () => {
stopMicCapture(activeRoverRef.current || roverId);
},
[roverId, stopMicCapture],
);
return ( return (
<section className={`surface ${flowWrapClass}`}> <section className={`surface ${flowWrapClass}`}>
<div className={innerFlowClass}> <div className={innerFlowClass}>
@@ -107,6 +405,23 @@ export default function VipAudioUploadCard({
Stop Stop
</button> </button>
</div> </div>
<div className="surface-muted mx-auto flex w-full max-w-sm flex-col gap-0.5 p-0.5 text-xs text-slate-300 text-center">
<p className="text-slate-200">Microphone Forwarding</p>
<label className="flex items-center justify-center gap-0.5">
<input
type="checkbox"
checked={openMicEnabled}
disabled={!roverId}
onChange={(event) => setOpenMicEnabled(Boolean(event.target.checked))}
/>
<span>Open mic</span>
</label>
<p className="text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'm'} (hold)</p>
<p className="text-slate-400">mic: {micState}</p>
<p className="text-slate-500">transport: whip</p>
</div>
{selectedForwardState ? ( {selectedForwardState ? (
<div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center"> <div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center">
state: {selectedForwardState.state || 'idle'} state: {selectedForwardState.state || 'idle'}
webui/src/context/SessionContext.jsx
+6
View File
@@ -28,6 +28,9 @@ const SessionContext = createContext({
rebootServer: async () => {}, rebootServer: async () => {},
playUploadedAudio: async () => {}, playUploadedAudio: async () => {},
stopUploadedAudio: async () => {}, stopUploadedAudio: async () => {},
startMicWhip: async () => {},
readyMicWhip: async () => {},
stopMicWhip: async () => {},
setAudioLevels: async () => {}, setAudioLevels: async () => {},
llmControl: async () => {}, llmControl: async () => {},
}); });
@@ -147,6 +150,9 @@ export function SessionProvider({ children }) {
playUploadedAudio: ({ roverId, name, mime, dataBase64 }) => playUploadedAudio: ({ roverId, name, mime, dataBase64 }) =>
emitWithAck('audio:uploadPlay', { roverId, name, mime, dataBase64 }), emitWithAck('audio:uploadPlay', { roverId, name, mime, dataBase64 }),
stopUploadedAudio: (roverId) => emitWithAck('audio:uploadStop', { roverId }), stopUploadedAudio: (roverId) => emitWithAck('audio:uploadStop', { roverId }),
startMicWhip: (roverId) => emitWithAck('audio:micWhipStart', { roverId }),
readyMicWhip: (roverId) => emitWithAck('audio:micWhipReady', { roverId }),
stopMicWhip: (roverId) => emitWithAck('audio:micWhipStop', { roverId }),
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels), setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) => llmControl: (action, controls = {}) =>
emitWithAck('llm:control', { controls: { action, ...controls } }), emitWithAck('llm:control', { controls: { action, ...controls } }),
webui/src/controls/ControlContext.jsx
+6
View File
@@ -452,6 +452,10 @@ export function ControlSystemProvider({ children }) {
dispatch({ type: 'control/register-input-state', payload: { source, state: data } }); dispatch({ type: 'control/register-input-state', payload: { source, state: data } });
}, []); }, []);
const setMicPttActive = useCallback((active) => {
dispatch({ type: 'control/set-mic-ptt', payload: Boolean(active) });
}, []);
const contextValue = useMemo( const contextValue = useMemo(
() => ({ () => ({
state, state,
@@ -478,6 +482,7 @@ export function ControlSystemProvider({ children }) {
sendSong, sendSong,
startHorn, startHorn,
stopHorn, stopHorn,
setMicPttActive,
}, },
}), }),
[ [
@@ -503,6 +508,7 @@ export function ControlSystemProvider({ children }) {
sendSong, sendSong,
startHorn, startHorn,
stopHorn, stopHorn,
setMicPttActive,
], ],
); );
webui/src/controls/constants.js
+1
View File
@@ -48,6 +48,7 @@ export const DEFAULT_KEYMAP = {
cameraDown: ['j'], cameraDown: ['j'],
nightVisionToggle: ['e'], nightVisionToggle: ['e'],
hornHonk: ['h'], hornHonk: ['h'],
micPtt: ['m'],
driveMacro: ['f'], driveMacro: ['f'],
dockMacro: ['g'], dockMacro: ['g'],
chatFocus: ['enter'], chatFocus: ['enter'],
webui/src/controls/controlReducer.js
+17
View File
@@ -35,6 +35,12 @@ function createHornState() {
}; };
} }
function createMicState() {
return {
pttActive: false,
};
}
export const initialControlState = { export const initialControlState = {
roverId: null, roverId: null,
mode: 'drive', mode: 'drive',
@@ -43,6 +49,7 @@ export const initialControlState = {
camera: createCameraState(), camera: createCameraState(),
song: createSongState(), song: createSongState(),
horn: createHornState(), horn: createHornState(),
mic: createMicState(),
lastControlIntentAt: 0, lastControlIntentAt: 0,
macros: DEFAULT_MACROS, macros: DEFAULT_MACROS,
keymap: DEFAULT_KEYMAP, keymap: DEFAULT_KEYMAP,
@@ -59,6 +66,7 @@ export function controlReducer(state, action) {
aux: action.payload ? state.aux : createAuxState(), aux: action.payload ? state.aux : createAuxState(),
song: action.payload ? state.song : createSongState(), song: action.payload ? state.song : createSongState(),
horn: action.payload ? state.horn : createHornState(), horn: action.payload ? state.horn : createHornState(),
mic: action.payload ? state.mic : createMicState(),
lastControlIntentAt: action.payload ? state.lastControlIntentAt : 0, lastControlIntentAt: action.payload ? state.lastControlIntentAt : 0,
}; };
case 'control/set-mode': case 'control/set-mode':
@@ -136,6 +144,7 @@ export function controlReducer(state, action) {
aux: createAuxState(), aux: createAuxState(),
song: createSongState(), song: createSongState(),
horn: createHornState(), horn: createHornState(),
mic: createMicState(),
lastControlIntentAt: 0, lastControlIntentAt: 0,
}; };
case 'control/set-horn-active': case 'control/set-horn-active':
@@ -168,6 +177,14 @@ export function controlReducer(state, action) {
note: action.payload ?? SONG_DEFAULT_NOTE, note: action.payload ?? SONG_DEFAULT_NOTE,
}, },
}; };
case 'control/set-mic-ptt':
return {
...state,
mic: {
...(state.mic || createMicState()),
pttActive: Boolean(action.payload),
},
};
default: default:
return state; return state;
} }
webui/src/controls/inputs/KeyboardInputManager.jsx
+10 -1
View File
@@ -129,6 +129,7 @@ export default function KeyboardInputManager() {
toggleNightVision, toggleNightVision,
startHorn, startHorn,
stopHorn, stopHorn,
setMicPttActive,
setSongNote, setSongNote,
sendSong, sendSong,
}, },
@@ -307,9 +308,10 @@ export default function KeyboardInputManager() {
hornActiveRef.current = false; hornActiveRef.current = false;
stopHorn(); stopHorn();
} }
setMicPttActive(false);
stopAllMotion(); stopAllMotion();
registerInputState(SOURCE, { keys: [], vector: ZERO_VECTOR, aux: ZERO_AUX }); registerInputState(SOURCE, { keys: [], vector: ZERO_VECTOR, aux: ZERO_AUX });
}, [registerInputState, stopAllMotion, stopHorn, stopServoLoop, stopSongLoop]); }, [registerInputState, setMicPttActive, stopAllMotion, stopHorn, stopServoLoop, stopSongLoop]);
const triggerHomeAssistantCycle = useCallback( const triggerHomeAssistantCycle = useCallback(
(targetState) => { (targetState) => {
@@ -378,6 +380,8 @@ export default function KeyboardInputManager() {
const started = startHorn(); const started = startHorn();
hornActiveRef.current = Boolean(started); hornActiveRef.current = Boolean(started);
} }
} else if (newlyPressed.some((token) => keymap.micPtt?.has(token))) {
setMicPttActive(true);
} else if (newlyPressed.some((token) => keymap.homeAssistantOn?.has(token))) { } else if (newlyPressed.some((token) => keymap.homeAssistantOn?.has(token))) {
triggerHomeAssistantCycle('on'); triggerHomeAssistantCycle('on');
} else if (newlyPressed.some((token) => keymap.homeAssistantOff?.has(token))) { } else if (newlyPressed.some((token) => keymap.homeAssistantOff?.has(token))) {
@@ -398,6 +402,9 @@ export default function KeyboardInputManager() {
hornActiveRef.current = false; hornActiveRef.current = false;
stopHorn(); stopHorn();
} }
if (!bindingActive(keymap.micPtt, activeTokensRef.current)) {
setMicPttActive(false);
}
ensureServoLoop(); ensureServoLoop();
ensureSongLoop(); ensureSongLoop();
driveFromKeys(); driveFromKeys();
@@ -427,8 +434,10 @@ export default function KeyboardInputManager() {
keymap.dockMacro, keymap.dockMacro,
keymap.driveMacro, keymap.driveMacro,
keymap.hornHonk, keymap.hornHonk,
keymap.micPtt,
resetAll, resetAll,
runMacro, runMacro,
setMicPttActive,
setMode, setMode,
stopAllMotion, stopAllMotion,
stopSongLoop, stopSongLoop,