legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-17 10:00:46 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

ound

Browse Source
This commit is contained in:
legop3
2026-03-17 19:46:53 -04:00
parent 93204ec1f1
commit 73ef6261ed
8 changed files with 204 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi/asound.conf
+2 -2
View File
@@ -12,8 +12,8 @@ pcm.dmixer {
rate 16000 rate 16000
channels 1 channels 1
period_time 0 period_time 0
period_size 1024 period_size 256
buffer_size 4096 buffer_size 1024
} }
} }
pi/bin/audio-forward-listener.sh
+5 -1
View File
@@ -41,6 +41,8 @@ run_pipeline() {
-loglevel warning \ -loglevel warning \
-fflags nobuffer \ -fflags nobuffer \
-flags low_delay \ -flags low_delay \
-max_delay 0 \
-reorder_queue_size 0 \
-analyzeduration 0 \ -analyzeduration 0 \
-probesize 32 \ -probesize 32 \
-i "${AUDIO_FORWARD_URL}" \ -i "${AUDIO_FORWARD_URL}" \
@@ -55,7 +57,9 @@ run_pipeline() {
-t raw \ -t raw \
-f S16_LE \ -f S16_LE \
-r 16000 \ -r 16000 \
-c 1 -c 1 \
-B 40000 \
-F 10000
local rc=$? local rc=$?
local -a statuses=("${PIPESTATUS[@]}") local -a statuses=("${PIPESTATUS[@]}")
LAST_FFMPEG_STATUS="${statuses[0]:-unknown}" LAST_FFMPEG_STATUS="${statuses[0]:-unknown}"
server/public/assets/index-CtTWAec1.js → server/public/assets/index-CXj15_94.js
+11 -11
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-CtTWAec1.js"></script> <script type="module" crossorigin src="/assets/index-CXj15_94.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css"> <link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head> </head>
<body> <body>
server/src/services/audioForwardService.js
+76 -70
View File
@@ -200,7 +200,7 @@ function buildPublisherArgs(fifoPath, outputUrl) {
'-application', '-application',
'lowdelay', 'lowdelay',
'-frame_duration', '-frame_duration',
'20', '10',
'-compression_level', '-compression_level',
'0', '0',
'-fflags', '-fflags',
@@ -258,30 +258,6 @@ function buildUploadWriterArgs(filePath) {
]; ];
} }
function buildMicWriterArgs() {
return [
'-hide_banner',
'-loglevel',
'warning',
'-fflags',
'nobuffer',
'-flags',
'low_delay',
'-i',
'pipe:0',
'-vn',
'-af',
'aresample=16000',
'-f',
's16le',
'-ac',
'1',
'-ar',
'16000',
'pipe:1',
];
}
function attachWriterPipe(worker, proc) { function attachWriterPipe(worker, proc) {
const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' }); const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' });
writer.on('error', (err) => { writer.on('error', (err) => {
@@ -315,20 +291,36 @@ function cleanupUploadFile(worker) {
} }
function stopContentWriter(worker) { function stopContentWriter(worker) {
if (!worker?.contentProc) return; if (!worker) return;
if (worker.micIdleTimer) { if (worker.micIdleTimer) {
clearTimeout(worker.micIdleTimer); clearTimeout(worker.micIdleTimer);
worker.micIdleTimer = null; worker.micIdleTimer = null;
} }
worker.micLastChunkAt = 0; worker.micLastChunkAt = 0;
if (worker.contentProc.stdin && !worker.contentProc.stdin.destroyed) { worker.micBackpressured = false;
if (worker.micWriter && !worker.micWriter.destroyed) {
try {
worker.micWriter.end();
} catch {
// noop
}
try {
worker.micWriter.destroy();
} catch {
// noop
}
}
worker.micWriter = null;
if (worker.contentProc && worker.contentProc.stdin && !worker.contentProc.stdin.destroyed) {
try { try {
worker.contentProc.stdin.destroy(); worker.contentProc.stdin.destroy();
} catch { } catch {
// noop // noop
} }
} }
stopProc(worker.contentProc); if (worker.contentProc) {
stopProc(worker.contentProc);
}
worker.contentProc = null; worker.contentProc = null;
worker.contentKind = null; worker.contentKind = null;
worker.activeOwnerSocketId = null; worker.activeOwnerSocketId = null;
@@ -423,68 +415,80 @@ function startMicWriter(roverId, ownerSocketId = null) {
stopContentWriter(worker); stopContentWriter(worker);
cleanupUploadFile(worker); cleanupUploadFile(worker);
const proc = spawnProcess(roverId, 'mic-writer', buildMicWriterArgs(), { const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' });
captureStdout: true, writer.on('error', (err) => {
captureStdin: true, const code = err?.code || 'unknown';
if (code !== 'EPIPE') {
logger.warn('mic fifo writer error', { roverId, code, message: err?.message || String(err) });
}
}); });
worker.contentProc = proc; writer.on('drain', () => {
const current = workers.get(roverId);
if (!current || current.contentKind !== 'mic') return;
current.micBackpressured = false;
});
writer.on('close', () => {
const current = workers.get(roverId);
if (!current || current.contentKind !== 'mic') return;
current.micWriter = null;
current.micBackpressured = false;
});
worker.micWriter = writer;
worker.micBackpressured = false;
worker.contentProc = null;
worker.contentKind = 'mic'; worker.contentKind = 'mic';
worker.activeOwnerSocketId = ownerSocketId; worker.activeOwnerSocketId = ownerSocketId;
worker.micLastChunkAt = Date.now(); worker.micLastChunkAt = Date.now();
scheduleMicIdleTimeout(roverId); scheduleMicIdleTimeout(roverId);
const seq = ++worker.writerSeq;
attachWriterPipe(worker, proc);
proc.stdin?.on('error', (err) => {
const code = err?.code || 'unknown';
if (code !== 'EPIPE') {
logger.warn('mic writer stdin error', { roverId, code, message: err?.message || String(err) });
}
});
setState(roverId, { state: 'playing', source: 'mic', error: null, startedAt: Date.now() }); setState(roverId, { state: 'playing', source: 'mic', error: null, startedAt: Date.now() });
proc.on('exit', (code, signal) => {
const current = workers.get(roverId);
if (!current || current.stopping) return;
if (current.writerSeq !== seq || current.contentProc !== proc) return;
current.contentProc = null;
current.contentKind = null;
current.activeOwnerSocketId = null;
if (current.micIdleTimer) {
clearTimeout(current.micIdleTimer);
current.micIdleTimer = null;
}
current.micLastChunkAt = 0;
if (code != null && code !== 0 && signal !== 'SIGTERM') {
setState(roverId, { state: 'error', source: 'mic', error: `mic writer exited code=${code} signal=${signal || 'none'}` });
}
startSilenceWriter(roverId);
});
} }
function pushMicChunk(roverId, ownerSocketId, dataBase64) { function decodeMicChunk(payload = {}) {
const binary = payload?.data;
if (Buffer.isBuffer(binary)) {
return binary;
}
if (binary instanceof Uint8Array) {
return Buffer.from(binary.buffer, binary.byteOffset, binary.byteLength);
}
if (binary instanceof ArrayBuffer) {
return Buffer.from(binary);
}
if (typeof payload?.dataBase64 === 'string' && payload.dataBase64.trim()) {
return Buffer.from(payload.dataBase64.trim(), 'base64');
}
return Buffer.alloc(0);
}
function pushMicChunk(roverId, ownerSocketId, payload = {}) {
const worker = workers.get(roverId); const worker = workers.get(roverId);
if (!worker) { if (!worker) {
throw new Error('Audio forward worker unavailable'); throw new Error('Audio forward worker unavailable');
} }
if (worker.contentKind !== 'mic' || !worker.contentProc || worker.activeOwnerSocketId !== ownerSocketId) { if (worker.contentKind !== 'mic' || !worker.micWriter || worker.activeOwnerSocketId !== ownerSocketId) {
throw new Error('Mic forwarding is not active'); throw new Error('Mic forwarding is not active');
} }
const encoded = typeof dataBase64 === 'string' ? dataBase64.trim() : ''; const bytes = decodeMicChunk(payload);
if (!encoded) { if (!bytes.length) {
throw new Error('Mic chunk missing'); throw new Error('Mic chunk missing');
} }
const bytes = Buffer.from(encoded, 'base64'); if (bytes.length > 64 * 1024) {
if (!bytes.length) { throw new Error('Mic chunk too large');
throw new Error('Mic chunk decode failed');
} }
if (worker.contentProc.stdin?.writable !== true) { if (worker.micWriter.writable !== true) {
throw new Error('Mic writer input is not writable'); throw new Error('Mic writer input is not writable');
} }
if (worker.micBackpressured || worker.micWriter.writableNeedDrain) {
// Preserve low latency by dropping stale mic packets instead of queueing.
return;
}
worker.micLastChunkAt = Date.now(); worker.micLastChunkAt = Date.now();
worker.contentProc.stdin.write(bytes); const wrote = worker.micWriter.write(bytes);
if (!wrote) {
worker.micBackpressured = true;
}
scheduleMicIdleTimeout(roverId); scheduleMicIdleTimeout(roverId);
} }
@@ -520,8 +524,10 @@ function ensureWorker(roverId) {
contentKind: null, contentKind: null,
activeOwnerSocketId: null, activeOwnerSocketId: null,
activeUploadPath: null, activeUploadPath: null,
micWriter: null,
micLastChunkAt: 0, micLastChunkAt: 0,
micIdleTimer: null, micIdleTimer: null,
micBackpressured: false,
writerSeq: 0, writerSeq: 0,
stopping: false, stopping: false,
}; };
@@ -691,7 +697,7 @@ io.on('connection', (socket) => {
try { try {
const normalized = String(payload?.roverId || '').trim(); const normalized = String(payload?.roverId || '').trim();
ensureAudioForwardPermission(socket, normalized); ensureAudioForwardPermission(socket, normalized);
pushMicChunk(normalized, socket.id, payload?.dataBase64); pushMicChunk(normalized, socket.id, payload);
if (typeof cb === 'function') cb({ success: true }); if (typeof cb === 'function') cb({ success: true });
} catch (err) { } catch (err) {
if (typeof cb === 'function') cb({ error: err.message }); if (typeof cb === 'function') cb({ error: err.message });
webui/src/components/vip/VipAudioForwardingCard.jsx
+106 -27
View File
@@ -8,6 +8,41 @@ import {
} from './constants.js'; } from './constants.js';
import { useControlSystem } from '../../controls/index.js'; import { useControlSystem } from '../../controls/index.js';
const TARGET_SAMPLE_RATE = 16000;
function downsampleTo16k(input, sampleRate) {
if (!input || !input.length) return new Float32Array(0);
if (sampleRate === TARGET_SAMPLE_RATE) return input;
if (!Number.isFinite(sampleRate) || sampleRate < TARGET_SAMPLE_RATE) return input;
const ratio = sampleRate / TARGET_SAMPLE_RATE;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const output = new Float32Array(outputLength);
let inOffset = 0;
for (let outIdx = 0; outIdx < outputLength; outIdx += 1) {
const nextOffset = Math.min(input.length, Math.round((outIdx + 1) * ratio));
let sum = 0;
let count = 0;
for (let i = inOffset; i < nextOffset; i += 1) {
sum += input[i];
count += 1;
}
output[outIdx] = count > 0 ? sum / count : 0;
inOffset = nextOffset;
}
return output;
}
function floatToInt16Bytes(floatSamples) {
const bytes = new Uint8Array(floatSamples.length * 2);
const view = new DataView(bytes.buffer);
for (let i = 0; i < floatSamples.length; i += 1) {
const sample = Math.max(-1, Math.min(1, floatSamples[i]));
const int16 = sample < 0 ? Math.round(sample * 0x8000) : Math.round(sample * 0x7fff);
view.setInt16(i * 2, int16, true);
}
return bytes;
}
export default function VipAudioForwardingCard({ export default function VipAudioForwardingCard({
roster = [], roster = [],
ownRoverId = '', ownRoverId = '',
@@ -24,8 +59,11 @@ export default function VipAudioForwardingCard({
const [openMicEnabled, setOpenMicEnabled] = useState(false); const [openMicEnabled, setOpenMicEnabled] = useState(false);
const [micState, setMicState] = useState('idle'); const [micState, setMicState] = useState('idle');
const [message, setMessage] = useState(''); const [message, setMessage] = useState('');
const recorderRef = useRef(null);
const streamRef = useRef(null); const streamRef = useRef(null);
const audioContextRef = useRef(null);
const mediaSourceRef = useRef(null);
const processorRef = useRef(null);
const sinkRef = useRef(null);
const micActiveRef = useRef(false); const micActiveRef = useRef(false);
const activeRoverRef = useRef(''); const activeRoverRef = useRef('');
const singleRoverId = roster.length === 1 ? roster[0].id : ''; const singleRoverId = roster.length === 1 ? roster[0].id : '';
@@ -93,14 +131,37 @@ export default function VipAudioForwardingCard({
micActiveRef.current = false; micActiveRef.current = false;
setMicState('idle'); setMicState('idle');
try { try {
const recorder = recorderRef.current; if (processorRef.current && mediaSourceRef.current) {
if (recorder && recorder.state !== 'inactive') { mediaSourceRef.current.disconnect(processorRef.current);
recorder.stop();
} }
} catch { } catch {
// noop // noop
} }
recorderRef.current = null; try {
if (processorRef.current && sinkRef.current) {
processorRef.current.disconnect(sinkRef.current);
}
} catch {
// noop
}
try {
if (sinkRef.current && audioContextRef.current?.destination) {
sinkRef.current.disconnect(audioContextRef.current.destination);
}
} catch {
// noop
}
if (audioContextRef.current) {
try {
await audioContextRef.current.close();
} catch {
// noop
}
}
processorRef.current = null;
mediaSourceRef.current = null;
sinkRef.current = null;
audioContextRef.current = null;
if (streamRef.current) { if (streamRef.current) {
try { try {
streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current.getTracks().forEach((track) => track.stop());
@@ -130,12 +191,10 @@ export default function VipAudioForwardingCard({
if (!navigator.mediaDevices?.getUserMedia) { if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('Microphone capture is not supported in this browser.'); throw new Error('Microphone capture is not supported in this browser.');
} }
if (typeof MediaRecorder === 'undefined') {
throw new Error('MediaRecorder is not supported in this browser.');
}
await stopMicCapture(target); await stopMicCapture(target);
setMicState('starting'); setMicState('starting');
let stream = null; let stream = null;
let audioContext = null;
try { try {
stream = await navigator.mediaDevices.getUserMedia({ stream = await navigator.mediaDevices.getUserMedia({
audio: { audio: {
@@ -147,29 +206,38 @@ export default function VipAudioForwardingCard({
}); });
streamRef.current = stream; streamRef.current = stream;
await startMicForward?.(target); await startMicForward?.(target);
const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
? 'audio/webm;codecs=opus' if (!AudioContextCtor) {
: 'audio/webm'; throw new Error('Web Audio API is not supported in this browser.');
const recorder = new MediaRecorder(stream, { mimeType, audioBitsPerSecond: 64000 }); }
recorderRef.current = recorder; audioContext = new AudioContextCtor({ latencyHint: 'interactive' });
audioContextRef.current = audioContext;
const source = audioContext.createMediaStreamSource(stream);
mediaSourceRef.current = source;
const processor = audioContext.createScriptProcessor(1024, 1, 1);
processorRef.current = processor;
const sink = audioContext.createGain();
sink.gain.value = 0;
sinkRef.current = sink;
micActiveRef.current = true; micActiveRef.current = true;
activeRoverRef.current = target; activeRoverRef.current = target;
recorder.ondataavailable = async (event) => { processor.onaudioprocess = (event) => {
try {
if (!micActiveRef.current || !event.data || event.data.size <= 0) return;
const buffer = await event.data.arrayBuffer();
const base64 = bytesToBase64(new Uint8Array(buffer));
sendMicChunk?.({ roverId: target, dataBase64: base64 });
} catch {
// noop
}
};
recorder.onstop = () => {
if (!micActiveRef.current) return; if (!micActiveRef.current) return;
micActiveRef.current = false; const input = event.inputBuffer?.getChannelData(0);
setMicState('idle'); if (!input || input.length === 0) return;
const downsampled = downsampleTo16k(input, audioContext.sampleRate);
if (!downsampled.length) return;
const pcmBytes = floatToInt16Bytes(downsampled);
sendMicChunk?.({ roverId: target, data: pcmBytes.buffer });
}; };
recorder.start(120);
source.connect(processor);
processor.connect(sink);
sink.connect(audioContext.destination);
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
setMicState('live'); setMicState('live');
} catch (err) { } catch (err) {
if (stream) { if (stream) {
@@ -179,7 +247,18 @@ export default function VipAudioForwardingCard({
// noop // noop
} }
} }
if (audioContext) {
try {
await audioContext.close();
} catch {
// noop
}
}
streamRef.current = null; streamRef.current = null;
audioContextRef.current = null;
mediaSourceRef.current = null;
processorRef.current = null;
sinkRef.current = null;
throw err; throw err;
} }
}, },
webui/src/context/SessionContext.jsx
+2 -2
View File
@@ -152,8 +152,8 @@ export function SessionProvider({ children }) {
stopUploadedAudio: (roverId) => emitWithAck('audio:uploadStop', { roverId }), stopUploadedAudio: (roverId) => emitWithAck('audio:uploadStop', { roverId }),
startMicForward: (roverId) => emitWithAck('audio:micStart', { roverId }), startMicForward: (roverId) => emitWithAck('audio:micStart', { roverId }),
stopMicForward: (roverId) => emitWithAck('audio:micStop', { roverId }), stopMicForward: (roverId) => emitWithAck('audio:micStop', { roverId }),
sendMicChunk: ({ roverId, dataBase64 }) => { sendMicChunk: ({ roverId, dataBase64, data }) => {
socket.emit('audio:micChunk', { roverId, dataBase64 }); socket.emit('audio:micChunk', { roverId, dataBase64, data });
}, },
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels), setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) => llmControl: (action, controls = {}) =>
webui/src/controls/constants.js
+1 -1
View File
@@ -48,7 +48,7 @@ export const DEFAULT_KEYMAP = {
cameraDown: ['j'], cameraDown: ['j'],
nightVisionToggle: ['e'], nightVisionToggle: ['e'],
hornHonk: ['h'], hornHonk: ['h'],
micPtt: ['v'], micPtt: ['m'],
driveMacro: ['f'], driveMacro: ['f'],
dockMacro: ['g'], dockMacro: ['g'],
chatFocus: ['enter'], chatFocus: ['enter'],