legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 01:21:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

whipwhep

Browse Source
This commit is contained in:
legop3
2026-03-20 22:29:54 -04:00
parent f364e6ac60
commit 76c17abdb6
6 changed files with 214 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi/bin/audio-forward-listener.sh
+2 -2
View File
@@ -41,8 +41,8 @@ run_pipeline() {
-loglevel warning \ -loglevel warning \
-fflags nobuffer \ -fflags nobuffer \
-flags low_delay \ -flags low_delay \
-analyzeduration 0 \ -analyzeduration 200k \
-probesize 32 \ -probesize 32k \
-i "${AUDIO_FORWARD_URL}" \ -i "${AUDIO_FORWARD_URL}" \
-vn \ -vn \
-ac 1 \ -ac 1 \
server/public/assets/index-D2iMrx7r.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-E6gJe5Xr.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-D2iMrx7r.js"></script> <script type="module" crossorigin src="/assets/index-E6gJe5Xr.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css"> <link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head> </head>
<body> <body>
server/src/services/audioForwardService.js
+4
View File
@@ -679,6 +679,10 @@ roverManager.managerEvents.on('rover', ({ roverId, action } = {}) => {
return; return;
} }
if (action === 'upsert' && serviceEnabled) { if (action === 'upsert' && serviceEnabled) {
if (whipOwners.has(roverId)) {
// WHIP publishes directly to the forward path; avoid recreating local publisher mid-session.
return;
}
try { try {
ensureWorker(roverId); ensureWorker(roverId);
} catch (err) { } catch (err) {
webui/src/components/vip/VipAudioForwardingCard.jsx
+80 -39
View File
@@ -103,6 +103,62 @@ function waitForPeerConnected(pc, timeoutMs = 10000) {
}); });
} }
async function configureSenderForLowLatency(sender) {
if (!sender?.getParameters || !sender?.setParameters) return;
const params = sender.getParameters() || {};
const first = (params.encodings && params.encodings[0]) || {};
params.encodings = [
{
...first,
maxBitrate: 64000,
dtx: 'disabled',
},
];
try {
await sender.setParameters(params);
} catch {
// Some browsers reject unsupported combinations; keep defaults.
}
}
function waitForOutboundAudioFlow(pc, timeoutMs = 6000) {
return new Promise((resolve, reject) => {
if (!pc) {
reject(new Error('Peer connection missing'));
return;
}
const start = Date.now();
let baseline = -1;
const timer = setInterval(async () => {
if (Date.now() - start > timeoutMs) {
clearInterval(timer);
reject(new Error('WHIP connected but no outbound audio flow'));
return;
}
try {
const senders = pc.getSenders().filter((s) => s.track?.kind === 'audio');
for (const sender of senders) {
const stats = await sender.getStats();
for (const report of stats.values()) {
if (report.type !== 'outbound-rtp' || report.kind !== 'audio') continue;
const sent = Number(report.bytesSent || 0);
const packets = Number(report.packetsSent || 0);
if (baseline < 0) {
baseline = sent;
} else if (sent > baseline + 200 || packets > 5) {
clearInterval(timer);
resolve();
return;
}
}
}
} catch {
// Keep polling until timeout.
}
}, 250);
});
}
function resampleTo16k(input, sampleRate) { function resampleTo16k(input, sampleRate) {
if (!input || !input.length) return new Float32Array(0); if (!input || !input.length) return new Float32Array(0);
if (sampleRate === TARGET_SAMPLE_RATE) return input; if (sampleRate === TARGET_SAMPLE_RATE) return input;
@@ -374,9 +430,9 @@ export default function VipAudioForwardingCard({
const pc = new RTCPeerConnection(RTC_CONFIG); const pc = new RTCPeerConnection(RTC_CONFIG);
whipPcRef.current = pc; whipPcRef.current = pc;
micTransportRef.current = 'whip'; micTransportRef.current = 'whip';
whipFailoverRef.current = false;
try { try {
stream.getAudioTracks().forEach((track) => pc.addTrack(track, stream)); const senders = stream.getAudioTracks().map((track) => pc.addTrack(track, stream));
await Promise.all(senders.map((sender) => configureSenderForLowLatency(sender)));
pc.onconnectionstatechange = () => { pc.onconnectionstatechange = () => {
const state = pc.connectionState; const state = pc.connectionState;
if (!micActiveRef.current) return; if (!micActiveRef.current) return;
@@ -384,24 +440,9 @@ export default function VipAudioForwardingCard({
setMicState('live'); setMicState('live');
return; return;
} }
if ((state === 'failed' || state === 'disconnected') && !whipFailoverRef.current) { if (state === 'failed' || state === 'disconnected') {
whipFailoverRef.current = true;
const roverId = activeRoverRef.current;
if (!roverId || !streamRef.current) return;
(async () => {
try {
await stopMicWhip?.(roverId);
} catch {
// noop
}
if (!micActiveRef.current || micTransportRef.current !== 'whip') return;
try {
await startSocketBridge(roverId, streamRef.current);
} catch (err) {
setMicState('error'); setMicState('error');
setMessage(err?.message || 'Mic fallback failed.'); setMessage(`WHIP transport ${state}.`);
}
})();
} }
}; };
@@ -423,6 +464,7 @@ export default function VipAudioForwardingCard({
const answerSdp = await response.text(); const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp }); await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
await waitForPeerConnected(pc, 10000); await waitForPeerConnected(pc, 10000);
await waitForOutboundAudioFlow(pc, 6000);
await readyMicWhip?.(target); await readyMicWhip?.(target);
setMicState('live'); setMicState('live');
} catch (err) { } catch (err) {
@@ -438,7 +480,7 @@ export default function VipAudioForwardingCard({
throw err; throw err;
} }
}, },
[readyMicWhip, startMicWhip, startSocketBridge, stopMicWhip], [readyMicWhip, startMicWhip],
); );
const startMicCapture = useCallback( const startMicCapture = useCallback(
@@ -458,30 +500,29 @@ export default function VipAudioForwardingCard({
audio: { audio: {
channelCount: 1, channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE, sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: true, echoCancellation: false,
noiseSuppression: true, noiseSuppression: false,
autoGainControl: true, autoGainControl: false,
}, },
}); });
const [track] = stream.getAudioTracks();
if (track?.applyConstraints) {
try {
await track.applyConstraints({
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
});
} catch {
// Constraint support varies by browser; use acquired track as-is.
}
}
streamRef.current = stream; streamRef.current = stream;
micActiveRef.current = true; micActiveRef.current = true;
activeRoverRef.current = target; activeRoverRef.current = target;
let whipErr = null;
try {
await startWhipBridge(target, stream); await startWhipBridge(target, stream);
return;
} catch (err) {
whipErr = err;
try {
await stopMicWhip?.(target);
} catch {
// noop
}
}
await startSocketBridge(target, stream);
if (whipErr) {
setMessage(`WHIP unavailable, using socket fallback: ${whipErr.message || 'unknown error'}`);
}
} catch (err) { } catch (err) {
if (stream) { if (stream) {
try { try {
@@ -500,7 +541,7 @@ export default function VipAudioForwardingCard({
throw err; throw err;
} }
}, },
[startSocketBridge, startWhipBridge, stopMicCapture, stopMicWhip], [startWhipBridge, stopMicCapture],
); );
useEffect(() => { useEffect(() => {