mirror of
https://github.com/legop3/MultiRoombaRover.git
synced 2026-09-15 17:12:59 -04:00
whipwhep
This commit is contained in:
legop3
@@ -1,29 +1,30 @@
|
||||
# private rovers
|
||||
## basic concept:
|
||||
private rovers will be mostly just for lockdown admins to drive and use, but they can be temporarily unlocked manually by lockdown admins for use by verified users.
|
||||
This means that locking / unlocking will act a little different than standard rovers.
|
||||
|
||||
- cannot be spectated by spectators, ever
|
||||
- cannot be replayed, ever
|
||||
- cannot be spectated by spectators, unless they are unlocked
|
||||
- cannot be replayed, unless they are unlocked
|
||||
- private status is defined in the roverd config
|
||||
- needs to never leak through access to anyone while locked
|
||||
- unlocking a private rover is a big deal for verified users
|
||||
- unlocking a private rover is a big deal for verified users (opening up a rover in the main living space for a special event)
|
||||
|
||||
## locking / unlocking:
|
||||
- private rovers start locked
|
||||
- when locked, only lockdown admins can drive them
|
||||
- when unlocked, only verified users can drive them
|
||||
- if left unlocked with no one online for 1 hour, the server will automatically lock them
|
||||
- when unlocked, only verified users (and lockdown admins of course) can drive them
|
||||
- if left unlocked with no one online for 30 mins, the server will automatically lock them
|
||||
|
||||
## cliff rules / speed limit / overcurrent limit
|
||||
### private rovers will be in a sensitive area, so their physical capabilities will be limited by the server
|
||||
- if the cliff sensors get triggered, stop the rover and back it up
|
||||
- speed limit is already kind of a thing but has never been tested, need to make sure it works all the way through the control pipeline
|
||||
- hard overcurrent limits done server-side. completely seperate from the current client only ones.
|
||||
- almost zero tolerance for wheel and side brush overcurrents
|
||||
- come up with a way to do this without making it feel too punishing. overcurrents often happen by accident
|
||||
- ignore the main brush, private rovers wont have one so it may read wrong
|
||||
### private rovers will be in a sensitive area, their physical capabilities will be optionally limited by the server, controllable by lockdown admins.
|
||||
- optional toggleable limits:
|
||||
- speed limit
|
||||
- hard overcurrent limiting (stop motor for a bit the instant it overcurrents for maybe 0.3s)
|
||||
- hard bump limits, stop and back up slightly on physical bumps of a certain short duration
|
||||
- cliff drops. back up and pause when any cliff sensor triggers, use their binary outputs for this as they are tuned well from factory.
|
||||
|
||||
## UI specifics
|
||||
- private rovers don't show in the spectator pages unless they are unlocked
|
||||
- private rovers don't show in the list for normal users unless they are unlocked
|
||||
- they will only show for lockdown admins
|
||||
- when unlocked, they show for everyone
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -11,7 +11,7 @@
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
|
||||
<title>Multi Roomba Rover</title>
|
||||
<script type="module" crossorigin src="/assets/index-DnqjOvSX.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-DcrX1ADu.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-Dx4QsRNa.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -8,10 +8,12 @@ const { loadConfig } = require('../helpers/configLoader');
|
||||
const roverManager = require('./roverManager');
|
||||
const turnService = require('./turnService');
|
||||
const { isVerified } = require('./verificationService');
|
||||
const videoSessions = require('./videoSessions');
|
||||
|
||||
const audioForwardEvents = new EventEmitter();
|
||||
const config = loadConfig();
|
||||
const audioForwardConfig = config.audioForward || {};
|
||||
const mediaConfig = config.media || {};
|
||||
const serviceEnabled = audioForwardConfig.enabled !== false;
|
||||
const ffmpegBin = audioForwardConfig.ffmpegBin || 'ffmpeg';
|
||||
const streamSuffix =
|
||||
@@ -26,6 +28,7 @@ const maxUploadBytes = Number.isFinite(audioForwardConfig.maxUploadBytes)
|
||||
|
||||
const states = new Map(); // roverId -> { state, source, error, startedAt, updatedAt }
|
||||
const workers = new Map(); // roverId -> worker
|
||||
const whipOwners = new Map(); // roverId -> socketId
|
||||
|
||||
function publishStateChange(roverId) {
|
||||
audioForwardEvents.emit('change', { roverId, state: states.get(roverId) || null });
|
||||
@@ -126,6 +129,29 @@ function resolveForwardUrl(roverId) {
|
||||
)},m=publish&latency=10&mode=caller&transtype=live&pkt_size=1316`;
|
||||
}
|
||||
|
||||
function resolveForwardPathId(roverId) {
|
||||
return `${roverId}${streamSuffix}`;
|
||||
}
|
||||
|
||||
function getMediaPrefix() {
|
||||
const base = mediaConfig.whepBaseUrl;
|
||||
if (!base) return '';
|
||||
try {
|
||||
const parsed = new URL(base);
|
||||
return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, '');
|
||||
} catch {
|
||||
return String(base).replace(/\/+$/, '');
|
||||
}
|
||||
}
|
||||
|
||||
function buildWhipUrl(pathId) {
|
||||
const prefix = getMediaPrefix();
|
||||
if (!prefix) {
|
||||
throw new Error('Server media base URL missing');
|
||||
}
|
||||
return `${prefix}/${encodeURIComponent(pathId)}/whip`;
|
||||
}
|
||||
|
||||
function spawnFfmpeg(roverId, tag, args, options = {}) {
|
||||
const proc = spawn(ffmpegBin, args, {
|
||||
stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'],
|
||||
@@ -399,6 +425,12 @@ function ensureWorker(roverId) {
|
||||
}
|
||||
|
||||
function stopWorker(roverId) {
|
||||
const whipOwner = whipOwners.get(roverId);
|
||||
if (whipOwner) {
|
||||
whipOwners.delete(roverId);
|
||||
revokeWhipSessionForRover(roverId, whipOwner);
|
||||
}
|
||||
|
||||
const worker = workers.get(roverId);
|
||||
if (!worker) return;
|
||||
|
||||
@@ -426,16 +458,12 @@ function writeUploadFile(roverId, payload = {}) {
|
||||
const { name, mime, dataBase64 } = payload || {};
|
||||
const ext = extFromUpload(name, mime);
|
||||
const encoded = typeof dataBase64 === 'string' ? dataBase64.trim() : '';
|
||||
if (!encoded) {
|
||||
throw new Error('Upload payload missing');
|
||||
}
|
||||
if (!encoded) throw new Error('Upload payload missing');
|
||||
|
||||
const bytes = Buffer.from(encoded, 'base64');
|
||||
if (!bytes.length) {
|
||||
throw new Error('Upload decode failed');
|
||||
}
|
||||
if (bytes.length > maxUploadBytes) {
|
||||
throw new Error(`Upload too large (max ${maxUploadBytes} bytes)`);
|
||||
}
|
||||
if (!bytes.length) throw new Error('Upload decode failed');
|
||||
if (bytes.length > maxUploadBytes) throw new Error(`Upload too large (max ${maxUploadBytes} bytes)`);
|
||||
|
||||
ensureRuntimeDir();
|
||||
const stem = sanitizeFileStem(name || `upload-${Date.now()}`);
|
||||
const filePath = path.join(uploadsDir, `${sanitizeRoverId(roverId)}-${Date.now()}-${stem}${ext}`);
|
||||
@@ -444,18 +472,52 @@ function writeUploadFile(roverId, payload = {}) {
|
||||
}
|
||||
|
||||
function playUploadedAudio(roverId, payload = {}, ownerSocketId = null) {
|
||||
stopWhipForRover(roverId, 'upload_override');
|
||||
ensureWorker(roverId);
|
||||
const uploadPath = writeUploadFile(roverId, payload);
|
||||
startUploadWriter(roverId, uploadPath, ownerSocketId);
|
||||
}
|
||||
|
||||
function stopPlayback(roverId) {
|
||||
stopWhipForRover(roverId, 'stop_playback');
|
||||
ensureWorker(roverId);
|
||||
startSilenceWriter(roverId);
|
||||
}
|
||||
|
||||
function stopOwnedUploadIfUnauthorized(roverId, ownerSocketId, reason = 'driver_change') {
|
||||
function revokeWhipSessionForRover(roverId, ownerSocketId) {
|
||||
if (!roverId || !ownerSocketId) return;
|
||||
const pathId = resolveForwardPathId(roverId);
|
||||
videoSessions.revokeWhere(
|
||||
(info) => info?.socketId === ownerSocketId && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
|
||||
);
|
||||
}
|
||||
|
||||
function stopWhipForRover(roverId, reason = 'unknown') {
|
||||
const ownerSocketId = whipOwners.get(roverId);
|
||||
if (!ownerSocketId) return;
|
||||
whipOwners.delete(roverId);
|
||||
revokeWhipSessionForRover(roverId, ownerSocketId);
|
||||
logger.info('Stopping WHIP mic session', { roverId, ownerSocketId, reason });
|
||||
try {
|
||||
ensureWorker(roverId);
|
||||
startSilenceWriter(roverId);
|
||||
} catch (err) {
|
||||
setState(roverId, { state: 'error', source: 'mic-whip', error: err?.message || String(err), startedAt: null });
|
||||
}
|
||||
}
|
||||
|
||||
function stopOwnedAudioIfUnauthorized(roverId, ownerSocketId, reason = 'driver_change') {
|
||||
if (!roverId || !ownerSocketId) return;
|
||||
|
||||
if (whipOwners.get(roverId) === ownerSocketId) {
|
||||
const ownerSocket = io.sockets.sockets.get(ownerSocketId);
|
||||
const ownerIsDriver = ownerSocket ? roverManager.isDriver(roverId, ownerSocket) : false;
|
||||
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
|
||||
if (!ownerIsDriver || !ownerCanDrive) {
|
||||
stopWhipForRover(roverId, reason);
|
||||
}
|
||||
}
|
||||
|
||||
const worker = workers.get(roverId);
|
||||
if (!worker || worker.contentKind !== 'upload' || worker.activeOwnerSocketId !== ownerSocketId) return;
|
||||
|
||||
@@ -464,11 +526,7 @@ function stopOwnedUploadIfUnauthorized(roverId, ownerSocketId, reason = 'driver_
|
||||
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
|
||||
if (ownerIsDriver && ownerCanDrive) return;
|
||||
|
||||
logger.info('Stopping upload audio due to ownership/driver change', {
|
||||
roverId,
|
||||
ownerSocketId,
|
||||
reason,
|
||||
});
|
||||
logger.info('Stopping upload audio due to ownership/driver change', { roverId, ownerSocketId, reason });
|
||||
startSilenceWriter(roverId);
|
||||
}
|
||||
|
||||
@@ -479,6 +537,9 @@ roverManager.managerEvents.on('rover', ({ roverId, action } = {}) => {
|
||||
return;
|
||||
}
|
||||
if (action === 'upsert' && serviceEnabled) {
|
||||
if (whipOwners.has(roverId)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ensureWorker(roverId);
|
||||
} catch (err) {
|
||||
@@ -490,15 +551,19 @@ roverManager.managerEvents.on('rover', ({ roverId, action } = {}) => {
|
||||
roverManager.managerEvents.on('driver', ({ socketId, roverId, action } = {}) => {
|
||||
if (!socketId || !roverId) return;
|
||||
if (action === 'remove' || action === 'add') {
|
||||
stopOwnedUploadIfUnauthorized(roverId, socketId, action);
|
||||
stopOwnedAudioIfUnauthorized(roverId, socketId, action);
|
||||
}
|
||||
});
|
||||
|
||||
turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => {
|
||||
if (!roverId) return;
|
||||
const whipOwner = whipOwners.get(roverId);
|
||||
if (whipOwner) {
|
||||
stopOwnedAudioIfUnauthorized(roverId, whipOwner, 'turn_change');
|
||||
}
|
||||
const worker = workers.get(roverId);
|
||||
if (!worker || worker.contentKind !== 'upload') return;
|
||||
stopOwnedUploadIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
|
||||
stopOwnedAudioIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
|
||||
});
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
@@ -528,12 +593,61 @@ io.on('connection', (socket) => {
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('audio:micWhipStart', ({ roverId } = {}, cb = () => {}) => {
|
||||
try {
|
||||
const normalized = String(roverId || '').trim();
|
||||
ensureAudioForwardPermission(socket, normalized);
|
||||
stopWorker(normalized);
|
||||
whipOwners.set(normalized, socket.id);
|
||||
const pathId = resolveForwardPathId(normalized);
|
||||
revokeWhipSessionForRover(normalized, socket.id);
|
||||
const token = videoSessions.createSession(socket, { type: 'roverMic', id: pathId });
|
||||
const whipUrl = buildWhipUrl(pathId);
|
||||
setState(normalized, { state: 'starting', source: 'mic-whip', error: null, startedAt: Date.now() });
|
||||
cb({ success: true, roverId: normalized, pathId, token, whipUrl });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('audio:micWhipReady', ({ roverId } = {}, cb = () => {}) => {
|
||||
try {
|
||||
const normalized = String(roverId || '').trim();
|
||||
ensureAudioForwardPermission(socket, normalized);
|
||||
if (whipOwners.get(normalized) !== socket.id) {
|
||||
throw new Error('WHIP session not owned by this client');
|
||||
}
|
||||
setState(normalized, { state: 'playing', source: 'mic-whip', error: null, startedAt: Date.now() });
|
||||
cb({ success: true, roverId: normalized });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('audio:micWhipStop', ({ roverId } = {}, cb = () => {}) => {
|
||||
try {
|
||||
const normalized = String(roverId || '').trim();
|
||||
ensureAudioForwardPermission(socket, normalized);
|
||||
if (whipOwners.get(normalized) && whipOwners.get(normalized) !== socket.id) {
|
||||
throw new Error('Mic forwarding is owned by another session');
|
||||
}
|
||||
stopWhipForRover(normalized, 'client_stop');
|
||||
cb({ success: true, roverId: normalized });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
workers.forEach((worker, roverId) => {
|
||||
if (!worker || worker.contentKind !== 'upload' || worker.activeOwnerSocketId !== socket.id) return;
|
||||
logger.info('Stopping owned upload audio due to socket disconnect', { roverId, socketId: socket.id });
|
||||
startSilenceWriter(roverId);
|
||||
});
|
||||
for (const [roverId, ownerSocketId] of whipOwners.entries()) {
|
||||
if (ownerSocketId !== socket.id) continue;
|
||||
stopWhipForRover(roverId, 'socket_disconnect');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ const logger = require('../globals/logger').child('videoAuth');
|
||||
const videoSessions = require('./videoSessions');
|
||||
const { getMode, MODES } = require('./modeManager');
|
||||
const { isAdmin, isLockdownAdmin, getRole } = require('./roleService');
|
||||
const { isVerified } = require('./verificationService');
|
||||
const turnService = require('./turnService');
|
||||
const roverManager = require('./roverManager');
|
||||
const { loadConfig } = require('../helpers/configLoader');
|
||||
const { getRequestIp, getSocketIp, isLocalNetwork } = require('../helpers/ipResolver');
|
||||
@@ -145,7 +147,9 @@ app.post('/mediamtx/auth', (req, res) => {
|
||||
}
|
||||
|
||||
const info = videoSessions.getSession(sessionId);
|
||||
const streamTypeMatches = info && info.sourceType === streamInfo.type;
|
||||
const streamTypeMatches =
|
||||
info &&
|
||||
(info.sourceType === streamInfo.type || (info.sourceType === 'roverMic' && streamInfo.type === 'rover'));
|
||||
if (!info || !streamTypeMatches || info.sourceId !== streamInfo.id) {
|
||||
logger.warn('invalid session %s for stream %s:%s', sessionId, streamInfo.type, streamInfo.id);
|
||||
return res.status(401).end();
|
||||
@@ -158,6 +162,20 @@ app.post('/mediamtx/auth', (req, res) => {
|
||||
if (!canView(socket)) {
|
||||
return res.status(401).end();
|
||||
}
|
||||
if (info.sourceType === 'roverMic' && action === 'publish') {
|
||||
const roverId = streamInfo.baseId || streamInfo.id;
|
||||
if (!isVerified(socket)) {
|
||||
return res.status(401).end();
|
||||
}
|
||||
if (!roverManager.isDriver(roverId, socket)) {
|
||||
return res.status(401).end();
|
||||
}
|
||||
if (!turnService.canDrive(roverId, socket)) {
|
||||
return res.status(401).end();
|
||||
}
|
||||
return res.status(200).end();
|
||||
}
|
||||
|
||||
const role = getRole(socket);
|
||||
const isAudio = streamInfo.id?.endsWith('-audio');
|
||||
if (role === 'spectator' && !isAdmin(socket) && !isAudio) {
|
||||
|
||||
@@ -24,6 +24,7 @@ const KEY_ACTIONS = [
|
||||
{ id: 'cameraDown', label: 'Camera Down', group: 'Camera' },
|
||||
{ id: 'nightVisionToggle', label: 'Toggle Night Vision', group: 'Camera' },
|
||||
{ id: 'hornHonk', label: 'Horn (Hold)', group: 'Audio' },
|
||||
{ id: 'micPtt', label: 'Mic Push To Talk', group: 'Audio' },
|
||||
{ id: 'driveMacro', label: 'Drive Macro', group: 'Macros' },
|
||||
{ id: 'dockMacro', label: 'Dock Macro', group: 'Macros' },
|
||||
{ id: 'chatFocus', label: 'Toggle Chat', group: 'Chat' },
|
||||
|
||||
@@ -7,7 +7,16 @@ import VipVerificationCard from './vip/VipVerificationCard.jsx';
|
||||
import VipIdentityCard from './vip/VipIdentityCard.jsx';
|
||||
|
||||
export default function VipPanel() {
|
||||
const { session, identifySession, requestVerification, playUploadedAudio, stopUploadedAudio } = useSession();
|
||||
const {
|
||||
session,
|
||||
identifySession,
|
||||
requestVerification,
|
||||
playUploadedAudio,
|
||||
stopUploadedAudio,
|
||||
startMicWhip,
|
||||
readyMicWhip,
|
||||
stopMicWhip,
|
||||
} = useSession();
|
||||
const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' });
|
||||
const { value: profile } = useSettingsNamespace('profile', { nickname: '' });
|
||||
|
||||
@@ -39,6 +48,9 @@ export default function VipPanel() {
|
||||
audioForwardByRover={session?.audioForward || {}}
|
||||
playUploadedAudio={playUploadedAudio}
|
||||
stopUploadedAudio={stopUploadedAudio}
|
||||
startMicWhip={startMicWhip}
|
||||
readyMicWhip={readyMicWhip}
|
||||
stopMicWhip={stopMicWhip}
|
||||
/>
|
||||
) : (
|
||||
<VipVerificationCard
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { fieldClass, flowWrapClass, innerFlowClass } from './constants.js';
|
||||
import { useControlSystem } from '../../controls/index.js';
|
||||
|
||||
const MAX_UPLOAD_BYTES = 8 * 1024 * 1024;
|
||||
const TARGET_SAMPLE_RATE = 16000;
|
||||
const RTC_CONFIG = {
|
||||
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
|
||||
bundlePolicy: 'max-bundle',
|
||||
rtcpMuxPolicy: 'require',
|
||||
};
|
||||
|
||||
function bytesToBase64(bytes) {
|
||||
let binary = '';
|
||||
@@ -13,16 +20,161 @@ function bytesToBase64(bytes) {
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function encodeBase64(value) {
|
||||
if (typeof btoa === 'function') return btoa(value);
|
||||
return '';
|
||||
}
|
||||
|
||||
function buildAuthHeader(token) {
|
||||
if (!token) return {};
|
||||
const encoded = encodeBase64(`${token}:${token}`);
|
||||
return encoded ? { Authorization: `Basic ${encoded}` } : {};
|
||||
}
|
||||
|
||||
function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
|
||||
return new Promise((resolve) => {
|
||||
if (!pc || pc.iceGatheringState === 'complete') {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
pc.removeEventListener('icegatheringstatechange', onChange);
|
||||
resolve();
|
||||
}, timeoutMs);
|
||||
function onChange() {
|
||||
if (pc.iceGatheringState === 'complete') {
|
||||
clearTimeout(timer);
|
||||
pc.removeEventListener('icegatheringstatechange', onChange);
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
pc.addEventListener('icegatheringstatechange', onChange);
|
||||
});
|
||||
}
|
||||
|
||||
function isPeerTransportReady(pc) {
|
||||
if (!pc) return false;
|
||||
const conn = pc.connectionState;
|
||||
const ice = pc.iceConnectionState;
|
||||
if (conn === 'connected') return true;
|
||||
if (ice === 'connected' || ice === 'completed') return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function waitForPeerConnected(pc, timeoutMs = 10000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!pc) {
|
||||
reject(new Error('Peer connection missing'));
|
||||
return;
|
||||
}
|
||||
if (isPeerTransportReady(pc)) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error('Peer connection timeout'));
|
||||
}, timeoutMs);
|
||||
const onState = () => {
|
||||
if (isPeerTransportReady(pc)) {
|
||||
cleanup();
|
||||
resolve();
|
||||
} else if (
|
||||
pc.connectionState === 'failed' ||
|
||||
pc.connectionState === 'closed' ||
|
||||
pc.iceConnectionState === 'failed'
|
||||
) {
|
||||
cleanup();
|
||||
reject(new Error(`Peer connection ${pc.connectionState || pc.iceConnectionState}`));
|
||||
}
|
||||
};
|
||||
function cleanup() {
|
||||
clearTimeout(timer);
|
||||
pc.removeEventListener('connectionstatechange', onState);
|
||||
pc.removeEventListener('iceconnectionstatechange', onState);
|
||||
}
|
||||
pc.addEventListener('connectionstatechange', onState);
|
||||
pc.addEventListener('iceconnectionstatechange', onState);
|
||||
});
|
||||
}
|
||||
|
||||
async function configureSenderForLowLatency(sender) {
|
||||
if (!sender?.getParameters || !sender?.setParameters) return;
|
||||
const params = sender.getParameters() || {};
|
||||
const first = (params.encodings && params.encodings[0]) || {};
|
||||
params.encodings = [
|
||||
{
|
||||
...first,
|
||||
maxBitrate: 64000,
|
||||
dtx: 'disabled',
|
||||
},
|
||||
];
|
||||
try {
|
||||
await sender.setParameters(params);
|
||||
} catch {
|
||||
// Browser support varies; keep defaults if rejected.
|
||||
}
|
||||
}
|
||||
|
||||
function waitForOutboundAudioFlow(pc, timeoutMs = 6000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!pc) {
|
||||
reject(new Error('Peer connection missing'));
|
||||
return;
|
||||
}
|
||||
const start = Date.now();
|
||||
let baseline = -1;
|
||||
const timer = setInterval(async () => {
|
||||
if (Date.now() - start > timeoutMs) {
|
||||
clearInterval(timer);
|
||||
reject(new Error('WHIP connected but no outbound audio flow'));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const senders = pc.getSenders().filter((s) => s.track?.kind === 'audio');
|
||||
for (const sender of senders) {
|
||||
const stats = await sender.getStats();
|
||||
for (const report of stats.values()) {
|
||||
if (report.type !== 'outbound-rtp' || report.kind !== 'audio') continue;
|
||||
const sent = Number(report.bytesSent || 0);
|
||||
const packets = Number(report.packetsSent || 0);
|
||||
if (baseline < 0) {
|
||||
baseline = sent;
|
||||
} else if (sent > baseline + 200 || packets > 5) {
|
||||
clearInterval(timer);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Keep polling until timeout.
|
||||
}
|
||||
}, 250);
|
||||
});
|
||||
}
|
||||
|
||||
export default function VipAudioUploadCard({
|
||||
ownRoverId = '',
|
||||
audioForwardByRover = {},
|
||||
playUploadedAudio,
|
||||
stopUploadedAudio,
|
||||
startMicWhip,
|
||||
readyMicWhip,
|
||||
stopMicWhip,
|
||||
}) {
|
||||
const { state: controlState } = useControlSystem();
|
||||
const roverId = String(ownRoverId || '').trim();
|
||||
const [selectedUpload, setSelectedUpload] = useState(null);
|
||||
const [working, setWorking] = useState(false);
|
||||
const [openMicEnabled, setOpenMicEnabled] = useState(false);
|
||||
const [micState, setMicState] = useState('idle');
|
||||
const [message, setMessage] = useState('');
|
||||
const streamRef = useRef(null);
|
||||
const whipPcRef = useRef(null);
|
||||
const micActiveRef = useRef(false);
|
||||
const activeRoverRef = useRef('');
|
||||
const pttActive = Boolean(controlState?.mic?.pttActive);
|
||||
|
||||
const selectedForwardState = useMemo(
|
||||
() => (roverId ? audioForwardByRover?.[roverId] || null : null),
|
||||
@@ -80,6 +232,152 @@ export default function VipAudioUploadCard({
|
||||
}
|
||||
};
|
||||
|
||||
const stopMicCapture = useCallback(
|
||||
async (targetRoverId) => {
|
||||
const target = String(targetRoverId || activeRoverRef.current || '').trim();
|
||||
micActiveRef.current = false;
|
||||
setMicState('idle');
|
||||
if (whipPcRef.current) {
|
||||
try {
|
||||
whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop());
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
try {
|
||||
whipPcRef.current.close();
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
}
|
||||
whipPcRef.current = null;
|
||||
if (streamRef.current) {
|
||||
try {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
}
|
||||
streamRef.current = null;
|
||||
if (target) {
|
||||
try {
|
||||
await stopMicWhip?.(target);
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
}
|
||||
activeRoverRef.current = '';
|
||||
},
|
||||
[stopMicWhip],
|
||||
);
|
||||
|
||||
const startWhipMic = useCallback(
|
||||
async (target) => {
|
||||
const startPayload = await startMicWhip?.(target);
|
||||
const whipUrl = String(startPayload?.whipUrl || '').trim();
|
||||
const token = String(startPayload?.token || '').trim();
|
||||
if (!whipUrl || !token) {
|
||||
throw new Error('WHIP endpoint unavailable');
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
sampleRate: TARGET_SAMPLE_RATE,
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
},
|
||||
});
|
||||
streamRef.current = stream;
|
||||
|
||||
const track = stream.getAudioTracks()?.[0];
|
||||
if (track?.applyConstraints) {
|
||||
try {
|
||||
await track.applyConstraints({
|
||||
channelCount: 1,
|
||||
sampleRate: TARGET_SAMPLE_RATE,
|
||||
echoCancellation: false,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
});
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
}
|
||||
|
||||
const pc = new RTCPeerConnection(RTC_CONFIG);
|
||||
whipPcRef.current = pc;
|
||||
stream.getAudioTracks().forEach((audioTrack) => {
|
||||
const sender = pc.addTrack(audioTrack, stream);
|
||||
configureSenderForLowLatency(sender);
|
||||
});
|
||||
|
||||
const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false });
|
||||
await pc.setLocalDescription(offer);
|
||||
await waitForIceGatheringComplete(pc, 1800);
|
||||
|
||||
const response = await fetch(whipUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/sdp',
|
||||
...buildAuthHeader(token),
|
||||
},
|
||||
body: pc.localDescription?.sdp || offer.sdp,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`WHIP request failed: ${response.status}`);
|
||||
}
|
||||
const answerSdp = await response.text();
|
||||
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
|
||||
await waitForPeerConnected(pc, 10000);
|
||||
await waitForOutboundAudioFlow(pc, 6000);
|
||||
await readyMicWhip?.(target);
|
||||
},
|
||||
[readyMicWhip, startMicWhip],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const desiredActive = Boolean(openMicEnabled || pttActive);
|
||||
let cancelled = false;
|
||||
|
||||
async function syncMicState() {
|
||||
if (!roverId || !desiredActive) {
|
||||
await stopMicCapture(roverId);
|
||||
return;
|
||||
}
|
||||
if (micActiveRef.current && activeRoverRef.current === roverId) return;
|
||||
|
||||
try {
|
||||
setMicState('starting');
|
||||
setMessage('');
|
||||
micActiveRef.current = true;
|
||||
activeRoverRef.current = roverId;
|
||||
await startWhipMic(roverId);
|
||||
if (!cancelled) {
|
||||
setMicState('live');
|
||||
}
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
setMicState('error');
|
||||
setMessage(err?.message || 'Failed to start mic forwarding.');
|
||||
}
|
||||
await stopMicCapture(roverId);
|
||||
}
|
||||
}
|
||||
|
||||
syncMicState();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
|
||||
|
||||
useEffect(
|
||||
() => () => {
|
||||
stopMicCapture(activeRoverRef.current || roverId);
|
||||
},
|
||||
[roverId, stopMicCapture],
|
||||
);
|
||||
|
||||
return (
|
||||
<section className={`surface ${flowWrapClass}`}>
|
||||
<div className={innerFlowClass}>
|
||||
@@ -107,6 +405,23 @@ export default function VipAudioUploadCard({
|
||||
Stop
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="surface-muted mx-auto flex w-full max-w-sm flex-col gap-0.5 p-0.5 text-xs text-slate-300 text-center">
|
||||
<p className="text-slate-200">Microphone Forwarding</p>
|
||||
<label className="flex items-center justify-center gap-0.5">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={openMicEnabled}
|
||||
disabled={!roverId}
|
||||
onChange={(event) => setOpenMicEnabled(Boolean(event.target.checked))}
|
||||
/>
|
||||
<span>Open mic</span>
|
||||
</label>
|
||||
<p className="text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'm'} (hold)</p>
|
||||
<p className="text-slate-400">mic: {micState}</p>
|
||||
<p className="text-slate-500">transport: whip</p>
|
||||
</div>
|
||||
|
||||
{selectedForwardState ? (
|
||||
<div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center">
|
||||
state: {selectedForwardState.state || 'idle'}
|
||||
|
||||
@@ -28,6 +28,9 @@ const SessionContext = createContext({
|
||||
rebootServer: async () => {},
|
||||
playUploadedAudio: async () => {},
|
||||
stopUploadedAudio: async () => {},
|
||||
startMicWhip: async () => {},
|
||||
readyMicWhip: async () => {},
|
||||
stopMicWhip: async () => {},
|
||||
setAudioLevels: async () => {},
|
||||
llmControl: async () => {},
|
||||
});
|
||||
@@ -147,6 +150,9 @@ export function SessionProvider({ children }) {
|
||||
playUploadedAudio: ({ roverId, name, mime, dataBase64 }) =>
|
||||
emitWithAck('audio:uploadPlay', { roverId, name, mime, dataBase64 }),
|
||||
stopUploadedAudio: (roverId) => emitWithAck('audio:uploadStop', { roverId }),
|
||||
startMicWhip: (roverId) => emitWithAck('audio:micWhipStart', { roverId }),
|
||||
readyMicWhip: (roverId) => emitWithAck('audio:micWhipReady', { roverId }),
|
||||
stopMicWhip: (roverId) => emitWithAck('audio:micWhipStop', { roverId }),
|
||||
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
|
||||
llmControl: (action, controls = {}) =>
|
||||
emitWithAck('llm:control', { controls: { action, ...controls } }),
|
||||
|
||||
@@ -452,6 +452,10 @@ export function ControlSystemProvider({ children }) {
|
||||
dispatch({ type: 'control/register-input-state', payload: { source, state: data } });
|
||||
}, []);
|
||||
|
||||
const setMicPttActive = useCallback((active) => {
|
||||
dispatch({ type: 'control/set-mic-ptt', payload: Boolean(active) });
|
||||
}, []);
|
||||
|
||||
const contextValue = useMemo(
|
||||
() => ({
|
||||
state,
|
||||
@@ -478,6 +482,7 @@ export function ControlSystemProvider({ children }) {
|
||||
sendSong,
|
||||
startHorn,
|
||||
stopHorn,
|
||||
setMicPttActive,
|
||||
},
|
||||
}),
|
||||
[
|
||||
@@ -503,6 +508,7 @@ export function ControlSystemProvider({ children }) {
|
||||
sendSong,
|
||||
startHorn,
|
||||
stopHorn,
|
||||
setMicPttActive,
|
||||
],
|
||||
);
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ export const DEFAULT_KEYMAP = {
|
||||
cameraDown: ['j'],
|
||||
nightVisionToggle: ['e'],
|
||||
hornHonk: ['h'],
|
||||
micPtt: ['m'],
|
||||
driveMacro: ['f'],
|
||||
dockMacro: ['g'],
|
||||
chatFocus: ['enter'],
|
||||
|
||||
@@ -35,6 +35,12 @@ function createHornState() {
|
||||
};
|
||||
}
|
||||
|
||||
function createMicState() {
|
||||
return {
|
||||
pttActive: false,
|
||||
};
|
||||
}
|
||||
|
||||
export const initialControlState = {
|
||||
roverId: null,
|
||||
mode: 'drive',
|
||||
@@ -43,6 +49,7 @@ export const initialControlState = {
|
||||
camera: createCameraState(),
|
||||
song: createSongState(),
|
||||
horn: createHornState(),
|
||||
mic: createMicState(),
|
||||
lastControlIntentAt: 0,
|
||||
macros: DEFAULT_MACROS,
|
||||
keymap: DEFAULT_KEYMAP,
|
||||
@@ -59,6 +66,7 @@ export function controlReducer(state, action) {
|
||||
aux: action.payload ? state.aux : createAuxState(),
|
||||
song: action.payload ? state.song : createSongState(),
|
||||
horn: action.payload ? state.horn : createHornState(),
|
||||
mic: action.payload ? state.mic : createMicState(),
|
||||
lastControlIntentAt: action.payload ? state.lastControlIntentAt : 0,
|
||||
};
|
||||
case 'control/set-mode':
|
||||
@@ -136,6 +144,7 @@ export function controlReducer(state, action) {
|
||||
aux: createAuxState(),
|
||||
song: createSongState(),
|
||||
horn: createHornState(),
|
||||
mic: createMicState(),
|
||||
lastControlIntentAt: 0,
|
||||
};
|
||||
case 'control/set-horn-active':
|
||||
@@ -168,6 +177,14 @@ export function controlReducer(state, action) {
|
||||
note: action.payload ?? SONG_DEFAULT_NOTE,
|
||||
},
|
||||
};
|
||||
case 'control/set-mic-ptt':
|
||||
return {
|
||||
...state,
|
||||
mic: {
|
||||
...(state.mic || createMicState()),
|
||||
pttActive: Boolean(action.payload),
|
||||
},
|
||||
};
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
|
||||
@@ -129,6 +129,7 @@ export default function KeyboardInputManager() {
|
||||
toggleNightVision,
|
||||
startHorn,
|
||||
stopHorn,
|
||||
setMicPttActive,
|
||||
setSongNote,
|
||||
sendSong,
|
||||
},
|
||||
@@ -307,9 +308,10 @@ export default function KeyboardInputManager() {
|
||||
hornActiveRef.current = false;
|
||||
stopHorn();
|
||||
}
|
||||
setMicPttActive(false);
|
||||
stopAllMotion();
|
||||
registerInputState(SOURCE, { keys: [], vector: ZERO_VECTOR, aux: ZERO_AUX });
|
||||
}, [registerInputState, stopAllMotion, stopHorn, stopServoLoop, stopSongLoop]);
|
||||
}, [registerInputState, setMicPttActive, stopAllMotion, stopHorn, stopServoLoop, stopSongLoop]);
|
||||
|
||||
const triggerHomeAssistantCycle = useCallback(
|
||||
(targetState) => {
|
||||
@@ -378,6 +380,8 @@ export default function KeyboardInputManager() {
|
||||
const started = startHorn();
|
||||
hornActiveRef.current = Boolean(started);
|
||||
}
|
||||
} else if (newlyPressed.some((token) => keymap.micPtt?.has(token))) {
|
||||
setMicPttActive(true);
|
||||
} else if (newlyPressed.some((token) => keymap.homeAssistantOn?.has(token))) {
|
||||
triggerHomeAssistantCycle('on');
|
||||
} else if (newlyPressed.some((token) => keymap.homeAssistantOff?.has(token))) {
|
||||
@@ -398,6 +402,9 @@ export default function KeyboardInputManager() {
|
||||
hornActiveRef.current = false;
|
||||
stopHorn();
|
||||
}
|
||||
if (!bindingActive(keymap.micPtt, activeTokensRef.current)) {
|
||||
setMicPttActive(false);
|
||||
}
|
||||
ensureServoLoop();
|
||||
ensureSongLoop();
|
||||
driveFromKeys();
|
||||
@@ -427,8 +434,10 @@ export default function KeyboardInputManager() {
|
||||
keymap.dockMacro,
|
||||
keymap.driveMacro,
|
||||
keymap.hornHonk,
|
||||
keymap.micPtt,
|
||||
resetAll,
|
||||
runMacro,
|
||||
setMicPttActive,
|
||||
setMode,
|
||||
stopAllMotion,
|
||||
stopSongLoop,
|
||||
|
||||
Reference in New Issue
Block a user