legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 01:21:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

whipwhep

Browse Source
This commit is contained in:
legop3
2026-03-20 01:43:06 -04:00
parent 5b8704ccb3
commit 7ad3dc7015
12 changed files with 529 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plans/to-do.md
+3 -3
View File
@@ -1,12 +1,12 @@
1. fix controls remapping [X] 1. fix controls remapping [X]
2. trusted user system 2. trusted user system [X]
3. private rovers 3. private rovers
4. home assistant switch that tells the server to force the lights on 4. home assistant switch that tells the server to force the lights on
5. color coding with colored names and tape 5. color coding with colored names and tape
6. light force-on switch 6. light force-on switch
7. audio forwarding 7. audio forwarding
- streaming from server to rovers - streaming from server to rovers [X]
- audio files first - audio files first [X]
- then voice chat - then voice chat
8. mobile controls column swapping (optional joystick on left) 8. mobile controls column swapping (optional joystick on left)
server/config.example.yaml
+5
View File
@@ -25,6 +25,11 @@ audioForward:
ffmpegBin: "ffmpeg" ffmpegBin: "ffmpeg"
# Optional stream suffix for fallback URL generation # Optional stream suffix for fallback URL generation
streamSuffix: "-fwd" streamSuffix: "-fwd"
# Browser mic ingress path suffix (WHIP publish target)
micSuffix: "-mic"
# Client should prefer WHIP and fall back to socket chunk streaming
micDefaultTransport: "whip"
micSocketFallback: true
# Max upload payload accepted via VIP forward upload # Max upload payload accepted via VIP forward upload
maxUploadBytes: 8388608 maxUploadBytes: 8388608
server/mediamtx/mediamtx.yml
-1
View File
@@ -33,7 +33,6 @@ srtAddress: :9000
authMethod: http authMethod: http
authHTTPAddress: http://127.0.0.1:8080/mediamtx/auth authHTTPAddress: http://127.0.0.1:8080/mediamtx/auth
authHTTPExclude: authHTTPExclude:
- action: publish
- action: api - action: api
- action: metrics - action: metrics
- action: pprof - action: pprof
server/public/assets/index-6qqVBdnk.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-CG8JYWgy.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-CG8JYWgy.js"></script> <script type="module" crossorigin src="/assets/index-6qqVBdnk.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css"> <link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head> </head>
<body> <body>
server/src/services/audioForwardService.js
+153 -4
View File
@@ -8,13 +8,16 @@ const { loadConfig } = require('../helpers/configLoader');
const roverManager = require('./roverManager'); const roverManager = require('./roverManager');
const { isVerified } = require('./verificationService'); const { isVerified } = require('./verificationService');
const turnService = require('./turnService'); const turnService = require('./turnService');
const videoSessions = require('./videoSessions');
const audioForwardEvents = new EventEmitter(); const audioForwardEvents = new EventEmitter();
const config = loadConfig(); const config = loadConfig();
const audioForwardConfig = config.audioForward || {}; const audioForwardConfig = config.audioForward || {};
const mediaConfig = config.media || {};
const serviceEnabled = audioForwardConfig.enabled !== false; const serviceEnabled = audioForwardConfig.enabled !== false;
const ffmpegBin = audioForwardConfig.ffmpegBin || 'ffmpeg'; const ffmpegBin = audioForwardConfig.ffmpegBin || 'ffmpeg';
const streamSuffix = typeof audioForwardConfig.streamSuffix === 'string' ? audioForwardConfig.streamSuffix : '-fwd'; const streamSuffix = typeof audioForwardConfig.streamSuffix === 'string' ? audioForwardConfig.streamSuffix : '-fwd';
const micSuffix = typeof audioForwardConfig.micSuffix === 'string' ? audioForwardConfig.micSuffix : '-mic';
const runtimeDir = path.resolve(audioForwardConfig.runtimeDir || '/tmp/mrr-audio-forward'); const runtimeDir = path.resolve(audioForwardConfig.runtimeDir || '/tmp/mrr-audio-forward');
const uploadsDir = path.join(runtimeDir, 'uploads'); const uploadsDir = path.join(runtimeDir, 'uploads');
const maxUploadBytes = Number.isFinite(audioForwardConfig.maxUploadBytes) const maxUploadBytes = Number.isFinite(audioForwardConfig.maxUploadBytes)
@@ -143,6 +146,34 @@ function resolveForwardUrl(roverId) {
return `srt://127.0.0.1:9000?streamid=#!::r=${encodeURIComponent(roverId + streamSuffix)},m=publish&latency=10&mode=caller&transtype=live&pkt_size=1316`; return `srt://127.0.0.1:9000?streamid=#!::r=${encodeURIComponent(roverId + streamSuffix)},m=publish&latency=10&mode=caller&transtype=live&pkt_size=1316`;
} }
function resolveMicPathId(roverId) {
return `${roverId}${micSuffix}`;
}
function resolveMicReadUrl(roverId) {
const pathId = resolveMicPathId(roverId);
return `srt://127.0.0.1:9000?streamid=#!::r=${encodeURIComponent(pathId)},m=request&latency=10&mode=caller&transtype=live&pkt_size=1316`;
}
function getMediaPrefix() {
const base = mediaConfig.whepBaseUrl;
if (!base) return '';
try {
const parsed = new URL(base);
return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, '');
} catch {
return String(base).replace(/\/+$/, '');
}
}
function buildWhipUrl(pathId) {
const prefix = getMediaPrefix();
if (!prefix) {
throw new Error('Server media base URL missing');
}
return `${prefix}/${encodeURIComponent(pathId)}/whip`;
}
function spawnProcess(roverId, tag, args, options = {}) { function spawnProcess(roverId, tag, args, options = {}) {
const proc = spawn(ffmpegBin, args, { const proc = spawn(ffmpegBin, args, {
stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'], stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'],
@@ -258,6 +289,34 @@ function buildUploadWriterArgs(filePath) {
]; ];
} }
function buildWhipRelayReaderArgs(inputUrl) {
return [
'-hide_banner',
'-loglevel',
'warning',
'-fflags',
'nobuffer',
'-flags',
'low_delay',
'-analyzeduration',
'0',
'-probesize',
'32',
'-i',
inputUrl,
'-vn',
'-af',
'aresample=16000',
'-f',
's16le',
'-ac',
'1',
'-ar',
'16000',
'pipe:1',
];
}
function attachWriterPipe(worker, proc) { function attachWriterPipe(worker, proc) {
const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' }); const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' });
writer.on('error', (err) => { writer.on('error', (err) => {
@@ -324,6 +383,7 @@ function stopContentWriter(worker) {
worker.contentProc = null; worker.contentProc = null;
worker.contentKind = null; worker.contentKind = null;
worker.activeOwnerSocketId = null; worker.activeOwnerSocketId = null;
worker.micWhipPathId = null;
} }
function startSilenceWriter(roverId) { function startSilenceWriter(roverId) {
@@ -407,7 +467,7 @@ function startMicWriter(roverId, ownerSocketId = null) {
if (!worker || worker.stopping) return; if (!worker || worker.stopping) return;
if ( if (
worker.contentKind === 'mic' && worker.contentKind === 'mic' &&
worker.contentProc && worker.micWriter &&
worker.activeOwnerSocketId === ownerSocketId worker.activeOwnerSocketId === ownerSocketId
) { ) {
return; return;
@@ -445,6 +505,50 @@ function startMicWriter(roverId, ownerSocketId = null) {
setState(roverId, { state: 'playing', source: 'mic', error: null, startedAt: Date.now() }); setState(roverId, { state: 'playing', source: 'mic', error: null, startedAt: Date.now() });
} }
function startMicWhipRelay(roverId, ownerSocketId = null) {
const worker = workers.get(roverId);
if (!worker || worker.stopping) return;
if (
worker.contentKind === 'mic_whip' &&
worker.contentProc &&
worker.activeOwnerSocketId === ownerSocketId
) {
return;
}
stopContentWriter(worker);
cleanupUploadFile(worker);
const inputUrl = resolveMicReadUrl(roverId);
const proc = spawnProcess(roverId, 'mic-whip-reader', buildWhipRelayReaderArgs(inputUrl), {
captureStdout: true,
});
worker.contentProc = proc;
worker.contentKind = 'mic_whip';
worker.activeOwnerSocketId = ownerSocketId;
worker.micWhipPathId = resolveMicPathId(roverId);
const seq = ++worker.writerSeq;
attachWriterPipe(worker, proc);
setState(roverId, { state: 'playing', source: 'mic-whip', error: null, startedAt: Date.now() });
proc.on('exit', (code, signal) => {
const current = workers.get(roverId);
if (!current || current.stopping) return;
if (current.writerSeq !== seq || current.contentProc !== proc) return;
current.contentProc = null;
current.contentKind = null;
current.activeOwnerSocketId = null;
current.micWhipPathId = null;
if (code != null && code !== 0 && signal !== 'SIGTERM') {
setState(roverId, {
state: 'error',
source: 'mic-whip',
error: `mic whip reader exited code=${code} signal=${signal || 'none'}`,
});
}
startSilenceWriter(roverId);
});
}
function decodeMicChunk(payload = {}) { function decodeMicChunk(payload = {}) {
const binary = payload?.data; const binary = payload?.data;
if (Buffer.isBuffer(binary)) { if (Buffer.isBuffer(binary)) {
@@ -531,6 +635,7 @@ function ensureWorker(roverId) {
activeOwnerSocketId: null, activeOwnerSocketId: null,
activeUploadPath: null, activeUploadPath: null,
micWriter: null, micWriter: null,
micWhipPathId: null,
micLastChunkAt: 0, micLastChunkAt: 0,
micIdleTimer: null, micIdleTimer: null,
micBackpressured: false, micBackpressured: false,
@@ -639,12 +744,18 @@ function stopOwnedAudioIfUnauthorized(roverId, ownerSocketId, reason = 'driver_c
if (!roverId || !ownerSocketId) return; if (!roverId || !ownerSocketId) return;
const worker = workers.get(roverId); const worker = workers.get(roverId);
if (!worker) return; if (!worker) return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic') return; if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic' && worker.contentKind !== 'mic_whip') return;
if (worker.activeOwnerSocketId !== ownerSocketId) return; if (worker.activeOwnerSocketId !== ownerSocketId) return;
const ownerSocket = io.sockets.sockets.get(ownerSocketId); const ownerSocket = io.sockets.sockets.get(ownerSocketId);
const ownerIsDriver = ownerSocket ? roverManager.isDriver(roverId, ownerSocket) : false; const ownerIsDriver = ownerSocket ? roverManager.isDriver(roverId, ownerSocket) : false;
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false; const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
if (ownerIsDriver && ownerCanDrive) return; if (ownerIsDriver && ownerCanDrive) return;
if (worker.contentKind === 'mic_whip' && worker.micWhipPathId) {
const pathId = worker.micWhipPathId;
videoSessions.revokeWhere(
(info) => info?.socketId === ownerSocketId && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
}
logger.info('Stopping audio forward due to ownership/driver change', { roverId, ownerSocketId, reason, source: worker.contentKind }); logger.info('Stopping audio forward due to ownership/driver change', { roverId, ownerSocketId, reason, source: worker.contentKind });
startSilenceWriter(roverId); startSilenceWriter(roverId);
} }
@@ -659,7 +770,7 @@ roverManager.managerEvents.on('driver', ({ socketId, roverId, action } = {}) =>
turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => { turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => {
if (!roverId) return; if (!roverId) return;
const worker = workers.get(roverId); const worker = workers.get(roverId);
if (!worker || (worker.contentKind !== 'upload' && worker.contentKind !== 'mic')) return; if (!worker || (worker.contentKind !== 'upload' && worker.contentKind !== 'mic' && worker.contentKind !== 'mic_whip')) return;
stopOwnedAudioIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change'); stopOwnedAudioIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
}); });
@@ -725,13 +836,51 @@ io.on('connection', (socket) => {
} }
}); });
socket.on('audio:micWhipStart', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
ensureWorker(normalized);
const pathId = resolveMicPathId(normalized);
videoSessions.revokeWhere(
(info) => info?.socketId === socket.id && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
startMicWhipRelay(normalized, socket.id);
const token = videoSessions.createSession(socket, { type: 'roverMic', id: pathId });
const whipUrl = buildWhipUrl(pathId);
cb({ success: true, roverId: normalized, pathId, token, whipUrl });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('audio:micWhipStop', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
const worker = workers.get(normalized);
if (worker && worker.contentKind === 'mic_whip' && worker.activeOwnerSocketId !== socket.id) {
throw new Error('Mic forwarding is owned by another session');
}
const pathId = resolveMicPathId(normalized);
videoSessions.revokeWhere(
(info) => info?.socketId === socket.id && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
stopPlayback(normalized);
cb({ success: true, roverId: normalized });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('disconnect', () => { socket.on('disconnect', () => {
workers.forEach((worker, roverId) => { workers.forEach((worker, roverId) => {
if (!worker || worker.activeOwnerSocketId !== socket.id) return; if (!worker || worker.activeOwnerSocketId !== socket.id) return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic') return; if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic' && worker.contentKind !== 'mic_whip') return;
logger.info('Stopping owned audio forward due to socket disconnect', { roverId, socketId: socket.id, source: worker.contentKind }); logger.info('Stopping owned audio forward due to socket disconnect', { roverId, socketId: socket.id, source: worker.contentKind });
startSilenceWriter(roverId); startSilenceWriter(roverId);
}); });
videoSessions.revokeWhere((info) => info?.socketId === socket.id && info?.sourceType === 'roverMic');
}); });
}); });
server/src/services/videoAuthService.js
+25
View File
@@ -4,6 +4,8 @@ const logger = require('../globals/logger').child('videoAuth');
const videoSessions = require('./videoSessions'); const videoSessions = require('./videoSessions');
const { getMode, MODES } = require('./modeManager'); const { getMode, MODES } = require('./modeManager');
const { isAdmin, isLockdownAdmin, getRole } = require('./roleService'); const { isAdmin, isLockdownAdmin, getRole } = require('./roleService');
const { isVerified } = require('./verificationService');
const turnService = require('./turnService');
const roverManager = require('./roverManager'); const roverManager = require('./roverManager');
const { loadConfig } = require('../helpers/configLoader'); const { loadConfig } = require('../helpers/configLoader');
const { getRequestIp, getSocketIp, isLocalNetwork } = require('../helpers/ipResolver'); const { getRequestIp, getSocketIp, isLocalNetwork } = require('../helpers/ipResolver');
@@ -48,6 +50,9 @@ function extractStreamInfo(path) {
const remaining = segments.slice(start, end); const remaining = segments.slice(start, end);
if (remaining.length === 1) { if (remaining.length === 1) {
const rawId = remaining[0] || ''; const rawId = remaining[0] || '';
if (rawId.endsWith('-mic')) {
return { type: 'roverMic', id: rawId, baseId: rawId.slice(0, -4) };
}
const baseId = rawId.endsWith('-audio') ? rawId.slice(0, -6) : rawId; const baseId = rawId.endsWith('-audio') ? rawId.slice(0, -6) : rawId;
return { type: 'rover', id: rawId, baseId }; return { type: 'rover', id: rawId, baseId };
} }
@@ -84,6 +89,9 @@ function extractStreamInfoFromBody(body = {}) {
extractSrtStreamId(body.query); extractSrtStreamId(body.query);
if (!srtId) return null; if (!srtId) return null;
if (srtId.endsWith('-mic')) {
return { type: 'roverMic', id: srtId, baseId: srtId.slice(0, -4) };
}
const baseId = srtId.endsWith('-audio') ? srtId.slice(0, -6) : srtId; const baseId = srtId.endsWith('-audio') ? srtId.slice(0, -6) : srtId;
return { type: 'rover', id: srtId, baseId }; return { type: 'rover', id: srtId, baseId };
} }
@@ -128,6 +136,10 @@ app.post('/mediamtx/auth', (req, res) => {
if ((action === 'read' && isSrtLikeProtocol) || isForwardAudioRead) { if ((action === 'read' && isSrtLikeProtocol) || isForwardAudioRead) {
return res.status(200).end(); return res.status(200).end();
} }
// Existing rover/media publishers use SRT without per-session tokens.
if (action === 'publish' && isSrtLikeProtocol) {
return res.status(200).end();
}
if (!sessionId || !streamInfo?.id) { if (!sessionId || !streamInfo?.id) {
logger.warn('auth missing session or stream (session=%s path=%s)', sessionId, path); logger.warn('auth missing session or stream (session=%s path=%s)', sessionId, path);
@@ -147,6 +159,19 @@ app.post('/mediamtx/auth', (req, res) => {
if (!canView(socket)) { if (!canView(socket)) {
return res.status(401).end(); return res.status(401).end();
} }
if (streamInfo.type === 'roverMic' && action === 'publish') {
const roverId = streamInfo.baseId || streamInfo.id;
if (!isVerified(socket)) {
return res.status(401).end();
}
if (!roverManager.isDriver(roverId, socket)) {
return res.status(401).end();
}
if (!turnService.canDrive(roverId, socket)) {
return res.status(401).end();
}
return res.status(200).end();
}
const role = getRole(socket); const role = getRole(socket);
const isAudio = streamInfo.id?.endsWith('-audio'); const isAudio = streamInfo.id?.endsWith('-audio');
if (role === 'spectator' && !isAdmin(socket) && !isAudio) { if (role === 'spectator' && !isAdmin(socket) && !isAudio) {
webui/src/components/VipPanel.jsx
+4
View File
@@ -16,6 +16,8 @@ export default function VipPanel() {
startMicForward, startMicForward,
stopMicForward, stopMicForward,
sendMicChunk, sendMicChunk,
startMicWhip,
stopMicWhip,
} = useSession(); } = useSession();
const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' }); const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' });
const { value: profile } = useSettingsNamespace('profile', { nickname: '' }); const { value: profile } = useSettingsNamespace('profile', { nickname: '' });
@@ -57,6 +59,8 @@ export default function VipPanel() {
startMicForward={startMicForward} startMicForward={startMicForward}
stopMicForward={stopMicForward} stopMicForward={stopMicForward}
sendMicChunk={sendMicChunk} sendMicChunk={sendMicChunk}
startMicWhip={startMicWhip}
stopMicWhip={stopMicWhip}
/> />
) : ( ) : (
<VipVerificationCard <VipVerificationCard
webui/src/components/vip/VipAudioForwardingCard.jsx
+208 -52
View File
@@ -11,6 +11,43 @@ import { useControlSystem } from '../../controls/index.js';
const TARGET_SAMPLE_RATE = 16000; const TARGET_SAMPLE_RATE = 16000;
const MIC_PACKET_MS = 40; const MIC_PACKET_MS = 40;
const MIC_PACKET_BYTES = (TARGET_SAMPLE_RATE * 2 * MIC_PACKET_MS) / 1000; // s16le mono const MIC_PACKET_BYTES = (TARGET_SAMPLE_RATE * 2 * MIC_PACKET_MS) / 1000; // s16le mono
const RTC_CONFIG = {
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
function encodeBase64(value) {
if (typeof btoa === 'function') return btoa(value);
return '';
}
function buildAuthHeader(token) {
if (!token) return {};
const encoded = encodeBase64(`${token}:${token}`);
return encoded ? { Authorization: `Basic ${encoded}` } : {};
}
function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
return new Promise((resolve) => {
if (!pc || pc.iceGatheringState === 'complete') {
resolve();
return;
}
const timer = setTimeout(() => {
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}, timeoutMs);
function onChange() {
if (pc.iceGatheringState === 'complete') {
clearTimeout(timer);
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}
}
pc.addEventListener('icegatheringstatechange', onChange);
});
}
function resampleTo16k(input, sampleRate) { function resampleTo16k(input, sampleRate) {
if (!input || !input.length) return new Float32Array(0); if (!input || !input.length) return new Float32Array(0);
@@ -61,6 +98,8 @@ export default function VipAudioForwardingCard({
startMicForward, startMicForward,
stopMicForward, stopMicForward,
sendMicChunk, sendMicChunk,
startMicWhip,
stopMicWhip,
}) { }) {
const { state: controlState } = useControlSystem(); const { state: controlState } = useControlSystem();
const [selectedUpload, setSelectedUpload] = useState(null); const [selectedUpload, setSelectedUpload] = useState(null);
@@ -75,6 +114,9 @@ export default function VipAudioForwardingCard({
const sinkRef = useRef(null); const sinkRef = useRef(null);
const pendingPcmChunksRef = useRef([]); const pendingPcmChunksRef = useRef([]);
const pendingPcmBytesRef = useRef(0); const pendingPcmBytesRef = useRef(0);
const whipPcRef = useRef(null);
const micTransportRef = useRef('none');
const whipFailoverRef = useRef(false);
const micActiveRef = useRef(false); const micActiveRef = useRef(false);
const activeRoverRef = useRef(''); const activeRoverRef = useRef('');
const singleRoverId = roster.length === 1 ? roster[0].id : ''; const singleRoverId = roster.length === 1 ? roster[0].id : '';
@@ -169,12 +211,27 @@ export default function VipAudioForwardingCard({
// noop // noop
} }
} }
if (whipPcRef.current) {
try {
whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop());
} catch {
// noop
}
try {
whipPcRef.current.close();
} catch {
// noop
}
}
whipPcRef.current = null;
processorRef.current = null; processorRef.current = null;
mediaSourceRef.current = null; mediaSourceRef.current = null;
sinkRef.current = null; sinkRef.current = null;
audioContextRef.current = null; audioContextRef.current = null;
pendingPcmChunksRef.current = []; pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0; pendingPcmBytesRef.current = 0;
whipFailoverRef.current = false;
micTransportRef.current = 'none';
if (streamRef.current) { if (streamRef.current) {
try { try {
streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current.getTracks().forEach((track) => track.stop());
@@ -184,6 +241,11 @@ export default function VipAudioForwardingCard({
} }
streamRef.current = null; streamRef.current = null;
if (target) { if (target) {
try {
await stopMicWhip?.(target);
} catch {
// noop
}
try { try {
await stopMicForward?.(target); await stopMicForward?.(target);
} catch { } catch {
@@ -192,7 +254,134 @@ export default function VipAudioForwardingCard({
} }
activeRoverRef.current = ''; activeRoverRef.current = '';
}, },
[stopMicForward], [stopMicForward, stopMicWhip],
);
const startSocketBridge = useCallback(
async (target, stream) => {
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
if (!AudioContextCtor) {
throw new Error('Web Audio API is not supported in this browser.');
}
await startMicForward?.(target);
const audioContext = new AudioContextCtor({ latencyHint: 'interactive' });
audioContextRef.current = audioContext;
const source = audioContext.createMediaStreamSource(stream);
mediaSourceRef.current = source;
const processor = audioContext.createScriptProcessor(1024, 1, 1);
processorRef.current = processor;
const sink = audioContext.createGain();
sink.gain.value = 0;
sinkRef.current = sink;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
micTransportRef.current = 'socket';
processor.onaudioprocess = (event) => {
if (!micActiveRef.current || micTransportRef.current !== 'socket') return;
const input = event.inputBuffer?.getChannelData(0);
if (!input || input.length === 0) return;
const resampled = resampleTo16k(input, audioContext.sampleRate);
if (!resampled.length) return;
const pcmBytes = floatToInt16Bytes(resampled);
pendingPcmChunksRef.current.push(pcmBytes);
pendingPcmBytesRef.current += pcmBytes.length;
while (pendingPcmBytesRef.current >= MIC_PACKET_BYTES) {
const merged = concatUint8(pendingPcmChunksRef.current, pendingPcmBytesRef.current);
const packet = merged.slice(0, MIC_PACKET_BYTES);
const rest = merged.slice(MIC_PACKET_BYTES);
pendingPcmChunksRef.current = rest.length ? [rest] : [];
pendingPcmBytesRef.current = rest.length;
sendMicChunk?.({ roverId: target, data: packet });
}
};
source.connect(processor);
processor.connect(sink);
sink.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
setMicState('live');
},
[sendMicChunk, startMicForward],
);
const startWhipBridge = useCallback(
async (target, stream) => {
const startPayload = await startMicWhip?.(target);
const whipUrl = String(startPayload?.whipUrl || '').trim();
const token = String(startPayload?.token || '').trim();
if (!whipUrl || !token) {
throw new Error('WHIP endpoint unavailable');
}
const pc = new RTCPeerConnection(RTC_CONFIG);
whipPcRef.current = pc;
micTransportRef.current = 'whip';
whipFailoverRef.current = false;
try {
stream.getAudioTracks().forEach((track) => pc.addTrack(track, stream));
pc.onconnectionstatechange = () => {
const state = pc.connectionState;
if (!micActiveRef.current) return;
if (state === 'connected') {
setMicState('live');
return;
}
if ((state === 'failed' || state === 'disconnected') && !whipFailoverRef.current) {
whipFailoverRef.current = true;
const roverId = activeRoverRef.current;
if (!roverId || !streamRef.current) return;
(async () => {
try {
await stopMicWhip?.(roverId);
} catch {
// noop
}
if (!micActiveRef.current || micTransportRef.current !== 'whip') return;
try {
await startSocketBridge(roverId, streamRef.current);
} catch (err) {
setMicState('error');
setMessage(err?.message || 'Mic fallback failed.');
}
})();
}
};
const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false });
await pc.setLocalDescription(offer);
await waitForIceGatheringComplete(pc, 1800);
const response = await fetch(whipUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/sdp',
...buildAuthHeader(token),
},
body: pc.localDescription?.sdp || offer.sdp,
});
if (!response.ok) {
throw new Error(`WHIP request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
setMicState('live');
} catch (err) {
try {
pc.close();
} catch {
// noop
}
if (whipPcRef.current === pc) {
whipPcRef.current = null;
}
micTransportRef.current = 'none';
throw err;
}
},
[startMicWhip, startSocketBridge, stopMicWhip],
); );
const startMicCapture = useCallback( const startMicCapture = useCallback(
@@ -207,7 +396,6 @@ export default function VipAudioForwardingCard({
await stopMicCapture(target); await stopMicCapture(target);
setMicState('starting'); setMicState('starting');
let stream = null; let stream = null;
let audioContext = null;
try { try {
stream = await navigator.mediaDevices.getUserMedia({ stream = await navigator.mediaDevices.getUserMedia({
audio: { audio: {
@@ -219,63 +407,28 @@ export default function VipAudioForwardingCard({
}, },
}); });
streamRef.current = stream; streamRef.current = stream;
await startMicForward?.(target);
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
if (!AudioContextCtor) {
throw new Error('Web Audio API is not supported in this browser.');
}
audioContext = new AudioContextCtor({ latencyHint: 'interactive' });
audioContextRef.current = audioContext;
const source = audioContext.createMediaStreamSource(stream);
mediaSourceRef.current = source;
const processor = audioContext.createScriptProcessor(1024, 1, 1);
processorRef.current = processor;
const sink = audioContext.createGain();
sink.gain.value = 0;
sinkRef.current = sink;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
micActiveRef.current = true; micActiveRef.current = true;
activeRoverRef.current = target; activeRoverRef.current = target;
processor.onaudioprocess = (event) => { let whipErr = null;
if (!micActiveRef.current) return; try {
const input = event.inputBuffer?.getChannelData(0); await startWhipBridge(target, stream);
if (!input || input.length === 0) return; return;
const resampled = resampleTo16k(input, audioContext.sampleRate); } catch (err) {
if (!resampled.length) return; whipErr = err;
const pcmBytes = floatToInt16Bytes(resampled);
pendingPcmChunksRef.current.push(pcmBytes);
pendingPcmBytesRef.current += pcmBytes.length;
while (pendingPcmBytesRef.current >= MIC_PACKET_BYTES) {
const merged = concatUint8(pendingPcmChunksRef.current, pendingPcmBytesRef.current);
const packet = merged.slice(0, MIC_PACKET_BYTES);
const rest = merged.slice(MIC_PACKET_BYTES);
pendingPcmChunksRef.current = rest.length ? [rest] : [];
pendingPcmBytesRef.current = rest.length;
sendMicChunk?.({ roverId: target, data: packet });
}
};
source.connect(processor);
processor.connect(sink);
sink.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
setMicState('live');
} catch (err) {
if (stream) {
try { try {
stream.getTracks().forEach((track) => track.stop()); await stopMicWhip?.(target);
} catch { } catch {
// noop // noop
} }
} }
if (audioContext) { await startSocketBridge(target, stream);
if (whipErr) {
setMessage(`WHIP unavailable, using socket fallback: ${whipErr.message || 'unknown error'}`);
}
} catch (err) {
if (stream) {
try { try {
await audioContext.close(); stream.getTracks().forEach((track) => track.stop());
} catch { } catch {
// noop // noop
} }
@@ -285,10 +438,12 @@ export default function VipAudioForwardingCard({
mediaSourceRef.current = null; mediaSourceRef.current = null;
processorRef.current = null; processorRef.current = null;
sinkRef.current = null; sinkRef.current = null;
whipPcRef.current = null;
micTransportRef.current = 'none';
throw err; throw err;
} }
}, },
[sendMicChunk, startMicForward, stopMicCapture], [startSocketBridge, startWhipBridge, stopMicCapture, stopMicWhip],
); );
useEffect(() => { useEffect(() => {
@@ -363,6 +518,7 @@ export default function VipAudioForwardingCard({
</label> </label>
<p className="text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'v'} (hold)</p> <p className="text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'v'} (hold)</p>
<p className="text-slate-400">mic: {micState}</p> <p className="text-slate-400">mic: {micState}</p>
<p className="text-slate-500">transport: {micTransportRef.current === 'none' ? 'idle' : micTransportRef.current}</p>
</div> </div>
{selectedForwardState ? ( {selectedForwardState ? (
<div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center"> <div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center">
webui/src/context/SessionContext.jsx
+4
View File
@@ -31,6 +31,8 @@ const SessionContext = createContext({
startMicForward: async () => {}, startMicForward: async () => {},
stopMicForward: async () => {}, stopMicForward: async () => {},
sendMicChunk: async () => {}, sendMicChunk: async () => {},
startMicWhip: async () => {},
stopMicWhip: async () => {},
setAudioLevels: async () => {}, setAudioLevels: async () => {},
llmControl: async () => {}, llmControl: async () => {},
}); });
@@ -161,6 +163,8 @@ export function SessionProvider({ children }) {
socket.emit('audio:micChunk', { roverId, dataBase64, data }); socket.emit('audio:micChunk', { roverId, dataBase64, data });
return true; return true;
}, },
startMicWhip: (roverId) => emitWithAck('audio:micWhipStart', { roverId }),
stopMicWhip: (roverId) => emitWithAck('audio:micWhipStop', { roverId }),
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels), setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) => llmControl: (action, controls = {}) =>
emitWithAck('llm:control', { controls: { action, ...controls } }), emitWithAck('llm:control', { controls: { action, ...controls } }),
webui/src/controls/constants.js
+1 -1
View File
@@ -62,7 +62,7 @@ export const DEFAULT_MACROS = [
{ {
id: 'drive-sequence', id: 'drive-sequence',
label: 'Drive', label: 'Drive',
description: 'Start, dock, and full command sequence used by the drive button.', description: 'Start, undock, and full command sequence used by the drive button.',
steps: [ steps: [
{ type: 'servo', angle: 0 }, { type: 'servo', angle: 0 },
{ type: 'oi', command: 'start' }, { type: 'oi', command: 'start' },