legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 09:31:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

whipwhep

Browse Source
This commit is contained in:
legop3
2026-03-20 01:43:06 -04:00
parent 5b8704ccb3
commit 7ad3dc7015
12 changed files with 529 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plans/to-do.md
+3 -3
View File
@@ -1,12 +1,12 @@
1. fix controls remapping [X]
2. trusted user system
2. trusted user system [X]
3. private rovers
4. home assistant switch that tells the server to force the lights on
5. color coding with colored names and tape
6. light force-on switch
7. audio forwarding
- streaming from server to rovers
- audio files first
- streaming from server to rovers [X]
- audio files first [X]
- then voice chat
8. mobile controls column swapping (optional joystick on left)
server/config.example.yaml
+5
View File
@@ -25,6 +25,11 @@ audioForward:
ffmpegBin: "ffmpeg"
# Optional stream suffix for fallback URL generation
streamSuffix: "-fwd"
# Browser mic ingress path suffix (WHIP publish target)
micSuffix: "-mic"
# Client should prefer WHIP and fall back to socket chunk streaming
micDefaultTransport: "whip"
micSocketFallback: true
# Max upload payload accepted via VIP forward upload
maxUploadBytes: 8388608
server/mediamtx/mediamtx.yml
-1
View File
@@ -33,7 +33,6 @@ srtAddress: :9000
authMethod: http
authHTTPAddress: http://127.0.0.1:8080/mediamtx/auth
authHTTPExclude:
- action: publish
- action: api
- action: metrics
- action: pprof
server/public/assets/index-6qqVBdnk.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-CG8JYWgy.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-CG8JYWgy.js"></script>
<script type="module" crossorigin src="/assets/index-6qqVBdnk.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head>
<body>
server/src/services/audioForwardService.js
+153 -4
View File
@@ -8,13 +8,16 @@ const { loadConfig } = require('../helpers/configLoader');
const roverManager = require('./roverManager');
const { isVerified } = require('./verificationService');
const turnService = require('./turnService');
const videoSessions = require('./videoSessions');
const audioForwardEvents = new EventEmitter();
const config = loadConfig();
const audioForwardConfig = config.audioForward || {};
const mediaConfig = config.media || {};
const serviceEnabled = audioForwardConfig.enabled !== false;
const ffmpegBin = audioForwardConfig.ffmpegBin || 'ffmpeg';
const streamSuffix = typeof audioForwardConfig.streamSuffix === 'string' ? audioForwardConfig.streamSuffix : '-fwd';
const micSuffix = typeof audioForwardConfig.micSuffix === 'string' ? audioForwardConfig.micSuffix : '-mic';
const runtimeDir = path.resolve(audioForwardConfig.runtimeDir || '/tmp/mrr-audio-forward');
const uploadsDir = path.join(runtimeDir, 'uploads');
const maxUploadBytes = Number.isFinite(audioForwardConfig.maxUploadBytes)
@@ -143,6 +146,34 @@ function resolveForwardUrl(roverId) {
return `srt://127.0.0.1:9000?streamid=#!::r=${encodeURIComponent(roverId + streamSuffix)},m=publish&latency=10&mode=caller&transtype=live&pkt_size=1316`;
}
function resolveMicPathId(roverId) {
return `${roverId}${micSuffix}`;
}
function resolveMicReadUrl(roverId) {
const pathId = resolveMicPathId(roverId);
return `srt://127.0.0.1:9000?streamid=#!::r=${encodeURIComponent(pathId)},m=request&latency=10&mode=caller&transtype=live&pkt_size=1316`;
}
function getMediaPrefix() {
const base = mediaConfig.whepBaseUrl;
if (!base) return '';
try {
const parsed = new URL(base);
return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, '');
} catch {
return String(base).replace(/\/+$/, '');
}
}
function buildWhipUrl(pathId) {
const prefix = getMediaPrefix();
if (!prefix) {
throw new Error('Server media base URL missing');
}
return `${prefix}/${encodeURIComponent(pathId)}/whip`;
}
function spawnProcess(roverId, tag, args, options = {}) {
const proc = spawn(ffmpegBin, args, {
stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'],
@@ -258,6 +289,34 @@ function buildUploadWriterArgs(filePath) {
];
}
function buildWhipRelayReaderArgs(inputUrl) {
return [
'-hide_banner',
'-loglevel',
'warning',
'-fflags',
'nobuffer',
'-flags',
'low_delay',
'-analyzeduration',
'0',
'-probesize',
'32',
'-i',
inputUrl,
'-vn',
'-af',
'aresample=16000',
'-f',
's16le',
'-ac',
'1',
'-ar',
'16000',
'pipe:1',
];
}
function attachWriterPipe(worker, proc) {
const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' });
writer.on('error', (err) => {
@@ -324,6 +383,7 @@ function stopContentWriter(worker) {
worker.contentProc = null;
worker.contentKind = null;
worker.activeOwnerSocketId = null;
worker.micWhipPathId = null;
}
function startSilenceWriter(roverId) {
@@ -407,7 +467,7 @@ function startMicWriter(roverId, ownerSocketId = null) {
if (!worker || worker.stopping) return;
if (
worker.contentKind === 'mic' &&
worker.contentProc &&
worker.micWriter &&
worker.activeOwnerSocketId === ownerSocketId
) {
return;
@@ -445,6 +505,50 @@ function startMicWriter(roverId, ownerSocketId = null) {
setState(roverId, { state: 'playing', source: 'mic', error: null, startedAt: Date.now() });
}
function startMicWhipRelay(roverId, ownerSocketId = null) {
const worker = workers.get(roverId);
if (!worker || worker.stopping) return;
if (
worker.contentKind === 'mic_whip' &&
worker.contentProc &&
worker.activeOwnerSocketId === ownerSocketId
) {
return;
}
stopContentWriter(worker);
cleanupUploadFile(worker);
const inputUrl = resolveMicReadUrl(roverId);
const proc = spawnProcess(roverId, 'mic-whip-reader', buildWhipRelayReaderArgs(inputUrl), {
captureStdout: true,
});
worker.contentProc = proc;
worker.contentKind = 'mic_whip';
worker.activeOwnerSocketId = ownerSocketId;
worker.micWhipPathId = resolveMicPathId(roverId);
const seq = ++worker.writerSeq;
attachWriterPipe(worker, proc);
setState(roverId, { state: 'playing', source: 'mic-whip', error: null, startedAt: Date.now() });
proc.on('exit', (code, signal) => {
const current = workers.get(roverId);
if (!current || current.stopping) return;
if (current.writerSeq !== seq || current.contentProc !== proc) return;
current.contentProc = null;
current.contentKind = null;
current.activeOwnerSocketId = null;
current.micWhipPathId = null;
if (code != null && code !== 0 && signal !== 'SIGTERM') {
setState(roverId, {
state: 'error',
source: 'mic-whip',
error: `mic whip reader exited code=${code} signal=${signal || 'none'}`,
});
}
startSilenceWriter(roverId);
});
}
function decodeMicChunk(payload = {}) {
const binary = payload?.data;
if (Buffer.isBuffer(binary)) {
@@ -531,6 +635,7 @@ function ensureWorker(roverId) {
activeOwnerSocketId: null,
activeUploadPath: null,
micWriter: null,
micWhipPathId: null,
micLastChunkAt: 0,
micIdleTimer: null,
micBackpressured: false,
@@ -639,12 +744,18 @@ function stopOwnedAudioIfUnauthorized(roverId, ownerSocketId, reason = 'driver_c
if (!roverId || !ownerSocketId) return;
const worker = workers.get(roverId);
if (!worker) return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic') return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic' && worker.contentKind !== 'mic_whip') return;
if (worker.activeOwnerSocketId !== ownerSocketId) return;
const ownerSocket = io.sockets.sockets.get(ownerSocketId);
const ownerIsDriver = ownerSocket ? roverManager.isDriver(roverId, ownerSocket) : false;
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
if (ownerIsDriver && ownerCanDrive) return;
if (worker.contentKind === 'mic_whip' && worker.micWhipPathId) {
const pathId = worker.micWhipPathId;
videoSessions.revokeWhere(
(info) => info?.socketId === ownerSocketId && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
}
logger.info('Stopping audio forward due to ownership/driver change', { roverId, ownerSocketId, reason, source: worker.contentKind });
startSilenceWriter(roverId);
}
@@ -659,7 +770,7 @@ roverManager.managerEvents.on('driver', ({ socketId, roverId, action } = {}) =>
turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => {
if (!roverId) return;
const worker = workers.get(roverId);
if (!worker || (worker.contentKind !== 'upload' && worker.contentKind !== 'mic')) return;
if (!worker || (worker.contentKind !== 'upload' && worker.contentKind !== 'mic' && worker.contentKind !== 'mic_whip')) return;
stopOwnedAudioIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
});
@@ -725,13 +836,51 @@ io.on('connection', (socket) => {
}
});
socket.on('audio:micWhipStart', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
ensureWorker(normalized);
const pathId = resolveMicPathId(normalized);
videoSessions.revokeWhere(
(info) => info?.socketId === socket.id && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
startMicWhipRelay(normalized, socket.id);
const token = videoSessions.createSession(socket, { type: 'roverMic', id: pathId });
const whipUrl = buildWhipUrl(pathId);
cb({ success: true, roverId: normalized, pathId, token, whipUrl });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('audio:micWhipStop', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
const worker = workers.get(normalized);
if (worker && worker.contentKind === 'mic_whip' && worker.activeOwnerSocketId !== socket.id) {
throw new Error('Mic forwarding is owned by another session');
}
const pathId = resolveMicPathId(normalized);
videoSessions.revokeWhere(
(info) => info?.socketId === socket.id && info?.sourceType === 'roverMic' && info?.sourceId === pathId,
);
stopPlayback(normalized);
cb({ success: true, roverId: normalized });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('disconnect', () => {
workers.forEach((worker, roverId) => {
if (!worker || worker.activeOwnerSocketId !== socket.id) return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic') return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic' && worker.contentKind !== 'mic_whip') return;
logger.info('Stopping owned audio forward due to socket disconnect', { roverId, socketId: socket.id, source: worker.contentKind });
startSilenceWriter(roverId);
});
videoSessions.revokeWhere((info) => info?.socketId === socket.id && info?.sourceType === 'roverMic');
});
});
server/src/services/videoAuthService.js
+25
View File
@@ -4,6 +4,8 @@ const logger = require('../globals/logger').child('videoAuth');
const videoSessions = require('./videoSessions');
const { getMode, MODES } = require('./modeManager');
const { isAdmin, isLockdownAdmin, getRole } = require('./roleService');
const { isVerified } = require('./verificationService');
const turnService = require('./turnService');
const roverManager = require('./roverManager');
const { loadConfig } = require('../helpers/configLoader');
const { getRequestIp, getSocketIp, isLocalNetwork } = require('../helpers/ipResolver');
@@ -48,6 +50,9 @@ function extractStreamInfo(path) {
const remaining = segments.slice(start, end);
if (remaining.length === 1) {
const rawId = remaining[0] || '';
if (rawId.endsWith('-mic')) {
return { type: 'roverMic', id: rawId, baseId: rawId.slice(0, -4) };
}
const baseId = rawId.endsWith('-audio') ? rawId.slice(0, -6) : rawId;
return { type: 'rover', id: rawId, baseId };
}
@@ -84,6 +89,9 @@ function extractStreamInfoFromBody(body = {}) {
extractSrtStreamId(body.query);
if (!srtId) return null;
if (srtId.endsWith('-mic')) {
return { type: 'roverMic', id: srtId, baseId: srtId.slice(0, -4) };
}
const baseId = srtId.endsWith('-audio') ? srtId.slice(0, -6) : srtId;
return { type: 'rover', id: srtId, baseId };
}
@@ -128,6 +136,10 @@ app.post('/mediamtx/auth', (req, res) => {
if ((action === 'read' && isSrtLikeProtocol) || isForwardAudioRead) {
return res.status(200).end();
}
// Existing rover/media publishers use SRT without per-session tokens.
if (action === 'publish' && isSrtLikeProtocol) {
return res.status(200).end();
}
if (!sessionId || !streamInfo?.id) {
logger.warn('auth missing session or stream (session=%s path=%s)', sessionId, path);
@@ -147,6 +159,19 @@ app.post('/mediamtx/auth', (req, res) => {
if (!canView(socket)) {
return res.status(401).end();
}
if (streamInfo.type === 'roverMic' && action === 'publish') {
const roverId = streamInfo.baseId || streamInfo.id;
if (!isVerified(socket)) {
return res.status(401).end();
}
if (!roverManager.isDriver(roverId, socket)) {
return res.status(401).end();
}
if (!turnService.canDrive(roverId, socket)) {
return res.status(401).end();
}
return res.status(200).end();
}
const role = getRole(socket);
const isAudio = streamInfo.id?.endsWith('-audio');
if (role === 'spectator' && !isAdmin(socket) && !isAudio) {
webui/src/components/VipPanel.jsx
+4
View File
@@ -16,6 +16,8 @@ export default function VipPanel() {
startMicForward,
stopMicForward,
sendMicChunk,
startMicWhip,
stopMicWhip,
} = useSession();
const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' });
const { value: profile } = useSettingsNamespace('profile', { nickname: '' });
@@ -57,6 +59,8 @@ export default function VipPanel() {
startMicForward={startMicForward}
stopMicForward={stopMicForward}
sendMicChunk={sendMicChunk}
startMicWhip={startMicWhip}
stopMicWhip={stopMicWhip}
/>
) : (
<VipVerificationCard
webui/src/components/vip/VipAudioForwardingCard.jsx
+208 -52
View File
@@ -11,6 +11,43 @@ import { useControlSystem } from '../../controls/index.js';
const TARGET_SAMPLE_RATE = 16000;
const MIC_PACKET_MS = 40;
const MIC_PACKET_BYTES = (TARGET_SAMPLE_RATE * 2 * MIC_PACKET_MS) / 1000; // s16le mono
const RTC_CONFIG = {
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
function encodeBase64(value) {
if (typeof btoa === 'function') return btoa(value);
return '';
}
function buildAuthHeader(token) {
if (!token) return {};
const encoded = encodeBase64(`${token}:${token}`);
return encoded ? { Authorization: `Basic ${encoded}` } : {};
}
function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
return new Promise((resolve) => {
if (!pc || pc.iceGatheringState === 'complete') {
resolve();
return;
}
const timer = setTimeout(() => {
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}, timeoutMs);
function onChange() {
if (pc.iceGatheringState === 'complete') {
clearTimeout(timer);
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}
}
pc.addEventListener('icegatheringstatechange', onChange);
});
}
function resampleTo16k(input, sampleRate) {
if (!input || !input.length) return new Float32Array(0);
@@ -61,6 +98,8 @@ export default function VipAudioForwardingCard({
startMicForward,
stopMicForward,
sendMicChunk,
startMicWhip,
stopMicWhip,
}) {
const { state: controlState } = useControlSystem();
const [selectedUpload, setSelectedUpload] = useState(null);
@@ -75,6 +114,9 @@ export default function VipAudioForwardingCard({
const sinkRef = useRef(null);
const pendingPcmChunksRef = useRef([]);
const pendingPcmBytesRef = useRef(0);
const whipPcRef = useRef(null);
const micTransportRef = useRef('none');
const whipFailoverRef = useRef(false);
const micActiveRef = useRef(false);
const activeRoverRef = useRef('');
const singleRoverId = roster.length === 1 ? roster[0].id : '';
@@ -169,12 +211,27 @@ export default function VipAudioForwardingCard({
// noop
}
}
if (whipPcRef.current) {
try {
whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop());
} catch {
// noop
}
try {
whipPcRef.current.close();
} catch {
// noop
}
}
whipPcRef.current = null;
processorRef.current = null;
mediaSourceRef.current = null;
sinkRef.current = null;
audioContextRef.current = null;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
whipFailoverRef.current = false;
micTransportRef.current = 'none';
if (streamRef.current) {
try {
streamRef.current.getTracks().forEach((track) => track.stop());
@@ -184,6 +241,11 @@ export default function VipAudioForwardingCard({
}
streamRef.current = null;
if (target) {
try {
await stopMicWhip?.(target);
} catch {
// noop
}
try {
await stopMicForward?.(target);
} catch {
@@ -192,7 +254,134 @@ export default function VipAudioForwardingCard({
}
activeRoverRef.current = '';
},
[stopMicForward],
[stopMicForward, stopMicWhip],
);
const startSocketBridge = useCallback(
async (target, stream) => {
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
if (!AudioContextCtor) {
throw new Error('Web Audio API is not supported in this browser.');
}
await startMicForward?.(target);
const audioContext = new AudioContextCtor({ latencyHint: 'interactive' });
audioContextRef.current = audioContext;
const source = audioContext.createMediaStreamSource(stream);
mediaSourceRef.current = source;
const processor = audioContext.createScriptProcessor(1024, 1, 1);
processorRef.current = processor;
const sink = audioContext.createGain();
sink.gain.value = 0;
sinkRef.current = sink;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
micTransportRef.current = 'socket';
processor.onaudioprocess = (event) => {
if (!micActiveRef.current || micTransportRef.current !== 'socket') return;
const input = event.inputBuffer?.getChannelData(0);
if (!input || input.length === 0) return;
const resampled = resampleTo16k(input, audioContext.sampleRate);
if (!resampled.length) return;
const pcmBytes = floatToInt16Bytes(resampled);
pendingPcmChunksRef.current.push(pcmBytes);
pendingPcmBytesRef.current += pcmBytes.length;
while (pendingPcmBytesRef.current >= MIC_PACKET_BYTES) {
const merged = concatUint8(pendingPcmChunksRef.current, pendingPcmBytesRef.current);
const packet = merged.slice(0, MIC_PACKET_BYTES);
const rest = merged.slice(MIC_PACKET_BYTES);
pendingPcmChunksRef.current = rest.length ? [rest] : [];
pendingPcmBytesRef.current = rest.length;
sendMicChunk?.({ roverId: target, data: packet });
}
};
source.connect(processor);
processor.connect(sink);
sink.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
setMicState('live');
},
[sendMicChunk, startMicForward],
);
const startWhipBridge = useCallback(
async (target, stream) => {
const startPayload = await startMicWhip?.(target);
const whipUrl = String(startPayload?.whipUrl || '').trim();
const token = String(startPayload?.token || '').trim();
if (!whipUrl || !token) {
throw new Error('WHIP endpoint unavailable');
}
const pc = new RTCPeerConnection(RTC_CONFIG);
whipPcRef.current = pc;
micTransportRef.current = 'whip';
whipFailoverRef.current = false;
try {
stream.getAudioTracks().forEach((track) => pc.addTrack(track, stream));
pc.onconnectionstatechange = () => {
const state = pc.connectionState;
if (!micActiveRef.current) return;
if (state === 'connected') {
setMicState('live');
return;
}
if ((state === 'failed' || state === 'disconnected') && !whipFailoverRef.current) {
whipFailoverRef.current = true;
const roverId = activeRoverRef.current;
if (!roverId || !streamRef.current) return;
(async () => {
try {
await stopMicWhip?.(roverId);
} catch {
// noop
}
if (!micActiveRef.current || micTransportRef.current !== 'whip') return;
try {
await startSocketBridge(roverId, streamRef.current);
} catch (err) {
setMicState('error');
setMessage(err?.message || 'Mic fallback failed.');
}
})();
}
};
const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false });
await pc.setLocalDescription(offer);
await waitForIceGatheringComplete(pc, 1800);
const response = await fetch(whipUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/sdp',
...buildAuthHeader(token),
},
body: pc.localDescription?.sdp || offer.sdp,
});
if (!response.ok) {
throw new Error(`WHIP request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
setMicState('live');
} catch (err) {
try {
pc.close();
} catch {
// noop
}
if (whipPcRef.current === pc) {
whipPcRef.current = null;
}
micTransportRef.current = 'none';
throw err;
}
},
[startMicWhip, startSocketBridge, stopMicWhip],
);
const startMicCapture = useCallback(
@@ -207,7 +396,6 @@ export default function VipAudioForwardingCard({
await stopMicCapture(target);
setMicState('starting');
let stream = null;
let audioContext = null;
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: {
@@ -219,63 +407,28 @@ export default function VipAudioForwardingCard({
},
});
streamRef.current = stream;
await startMicForward?.(target);
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
if (!AudioContextCtor) {
throw new Error('Web Audio API is not supported in this browser.');
}
audioContext = new AudioContextCtor({ latencyHint: 'interactive' });
audioContextRef.current = audioContext;
const source = audioContext.createMediaStreamSource(stream);
mediaSourceRef.current = source;
const processor = audioContext.createScriptProcessor(1024, 1, 1);
processorRef.current = processor;
const sink = audioContext.createGain();
sink.gain.value = 0;
sinkRef.current = sink;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
micActiveRef.current = true;
activeRoverRef.current = target;
processor.onaudioprocess = (event) => {
if (!micActiveRef.current) return;
const input = event.inputBuffer?.getChannelData(0);
if (!input || input.length === 0) return;
const resampled = resampleTo16k(input, audioContext.sampleRate);
if (!resampled.length) return;
const pcmBytes = floatToInt16Bytes(resampled);
pendingPcmChunksRef.current.push(pcmBytes);
pendingPcmBytesRef.current += pcmBytes.length;
while (pendingPcmBytesRef.current >= MIC_PACKET_BYTES) {
const merged = concatUint8(pendingPcmChunksRef.current, pendingPcmBytesRef.current);
const packet = merged.slice(0, MIC_PACKET_BYTES);
const rest = merged.slice(MIC_PACKET_BYTES);
pendingPcmChunksRef.current = rest.length ? [rest] : [];
pendingPcmBytesRef.current = rest.length;
sendMicChunk?.({ roverId: target, data: packet });
}
};
source.connect(processor);
processor.connect(sink);
sink.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
setMicState('live');
} catch (err) {
if (stream) {
let whipErr = null;
try {
await startWhipBridge(target, stream);
return;
} catch (err) {
whipErr = err;
try {
stream.getTracks().forEach((track) => track.stop());
await stopMicWhip?.(target);
} catch {
// noop
}
}
if (audioContext) {
await startSocketBridge(target, stream);
if (whipErr) {
setMessage(`WHIP unavailable, using socket fallback: ${whipErr.message || 'unknown error'}`);
}
} catch (err) {
if (stream) {
try {
await audioContext.close();
stream.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
@@ -285,10 +438,12 @@ export default function VipAudioForwardingCard({
mediaSourceRef.current = null;
processorRef.current = null;
sinkRef.current = null;
whipPcRef.current = null;
micTransportRef.current = 'none';
throw err;
}
},
[sendMicChunk, startMicForward, stopMicCapture],
[startSocketBridge, startWhipBridge, stopMicCapture, stopMicWhip],
);
useEffect(() => {
@@ -363,6 +518,7 @@ export default function VipAudioForwardingCard({
</label>
<p className="text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'v'} (hold)</p>
<p className="text-slate-400">mic: {micState}</p>
<p className="text-slate-500">transport: {micTransportRef.current === 'none' ? 'idle' : micTransportRef.current}</p>
</div>
{selectedForwardState ? (
<div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center">
webui/src/context/SessionContext.jsx
+4
View File
@@ -31,6 +31,8 @@ const SessionContext = createContext({
startMicForward: async () => {},
stopMicForward: async () => {},
sendMicChunk: async () => {},
startMicWhip: async () => {},
stopMicWhip: async () => {},
setAudioLevels: async () => {},
llmControl: async () => {},
});
@@ -161,6 +163,8 @@ export function SessionProvider({ children }) {
socket.emit('audio:micChunk', { roverId, dataBase64, data });
return true;
},
startMicWhip: (roverId) => emitWithAck('audio:micWhipStart', { roverId }),
stopMicWhip: (roverId) => emitWithAck('audio:micWhipStop', { roverId }),
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) =>
emitWithAck('llm:control', { controls: { action, ...controls } }),
webui/src/controls/constants.js
+1 -1
View File
@@ -62,7 +62,7 @@ export const DEFAULT_MACROS = [
{
id: 'drive-sequence',
label: 'Drive',
description: 'Start, dock, and full command sequence used by the drive button.',
description: 'Start, undock, and full command sequence used by the drive button.',
steps: [
{ type: 'servo', angle: 0 },
{ type: 'oi', command: 'start' },