legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-15 17:12:59 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

gwah

Browse Source
This commit is contained in:
legop3
2026-03-17 19:54:53 -04:00
parent 73ef6261ed
commit 5b8704ccb3
8 changed files with 183 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi/asound.conf
+2 -2
View File
@@ -12,8 +12,8 @@ pcm.dmixer {
rate 16000 rate 16000
channels 1 channels 1
period_time 0 period_time 0
period_size 256 period_size 1024
buffer_size 1024 buffer_size 4096
} }
} }
pi/bin/audio-forward-listener.sh
+1 -5
View File
@@ -41,8 +41,6 @@ run_pipeline() {
-loglevel warning \ -loglevel warning \
-fflags nobuffer \ -fflags nobuffer \
-flags low_delay \ -flags low_delay \
-max_delay 0 \
-reorder_queue_size 0 \
-analyzeduration 0 \ -analyzeduration 0 \
-probesize 32 \ -probesize 32 \
-i "${AUDIO_FORWARD_URL}" \ -i "${AUDIO_FORWARD_URL}" \
@@ -57,9 +55,7 @@ run_pipeline() {
-t raw \ -t raw \
-f S16_LE \ -f S16_LE \
-r 16000 \ -r 16000 \
-c 1 \ -c 1
-B 40000 \
-F 10000
local rc=$? local rc=$?
local -a statuses=("${PIPESTATUS[@]}") local -a statuses=("${PIPESTATUS[@]}")
LAST_FFMPEG_STATUS="${statuses[0]:-unknown}" LAST_FFMPEG_STATUS="${statuses[0]:-unknown}"
server/public/assets/index-CG8JYWgy.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-CXj15_94.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-CXj15_94.js"></script> <script type="module" crossorigin src="/assets/index-CG8JYWgy.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css"> <link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head> </head>
<body> <body>
server/src/services/audioForwardService.js
+6
View File
@@ -450,6 +450,9 @@ function decodeMicChunk(payload = {}) {
if (Buffer.isBuffer(binary)) { if (Buffer.isBuffer(binary)) {
return binary; return binary;
} }
if (binary && typeof binary === 'object' && binary.type === 'Buffer' && Array.isArray(binary.data)) {
return Buffer.from(binary.data);
}
if (binary instanceof Uint8Array) { if (binary instanceof Uint8Array) {
return Buffer.from(binary.buffer, binary.byteOffset, binary.byteLength); return Buffer.from(binary.buffer, binary.byteOffset, binary.byteLength);
} }
@@ -474,6 +477,9 @@ function pushMicChunk(roverId, ownerSocketId, payload = {}) {
if (!bytes.length) { if (!bytes.length) {
throw new Error('Mic chunk missing'); throw new Error('Mic chunk missing');
} }
if (bytes.length % 2 !== 0) {
throw new Error('Mic chunk has invalid PCM byte length');
}
if (bytes.length > 64 * 1024) { if (bytes.length > 64 * 1024) {
throw new Error('Mic chunk too large'); throw new Error('Mic chunk too large');
} }
webui/src/components/vip/VipAudioForwardingCard.jsx
+42 -16
View File
@@ -9,25 +9,24 @@ import {
import { useControlSystem } from '../../controls/index.js'; import { useControlSystem } from '../../controls/index.js';
const TARGET_SAMPLE_RATE = 16000; const TARGET_SAMPLE_RATE = 16000;
const MIC_PACKET_MS = 40;
const MIC_PACKET_BYTES = (TARGET_SAMPLE_RATE * 2 * MIC_PACKET_MS) / 1000; // s16le mono
function downsampleTo16k(input, sampleRate) { function resampleTo16k(input, sampleRate) {
if (!input || !input.length) return new Float32Array(0); if (!input || !input.length) return new Float32Array(0);
if (sampleRate === TARGET_SAMPLE_RATE) return input; if (sampleRate === TARGET_SAMPLE_RATE) return input;
if (!Number.isFinite(sampleRate) || sampleRate < TARGET_SAMPLE_RATE) return input; if (!Number.isFinite(sampleRate) || sampleRate <= 0) return input;
const ratio = sampleRate / TARGET_SAMPLE_RATE; const ratio = sampleRate / TARGET_SAMPLE_RATE;
const outputLength = Math.max(1, Math.round(input.length / ratio)); const outputLength = Math.max(1, Math.round(input.length / ratio));
const output = new Float32Array(outputLength); const output = new Float32Array(outputLength);
let inOffset = 0;
for (let outIdx = 0; outIdx < outputLength; outIdx += 1) { for (let outIdx = 0; outIdx < outputLength; outIdx += 1) {
const nextOffset = Math.min(input.length, Math.round((outIdx + 1) * ratio)); const src = outIdx * ratio;
let sum = 0; const srcFloor = Math.floor(src);
let count = 0; const srcCeil = Math.min(input.length - 1, srcFloor + 1);
for (let i = inOffset; i < nextOffset; i += 1) { const frac = src - srcFloor;
sum += input[i]; const a = input[srcFloor] ?? 0;
count += 1; const b = input[srcCeil] ?? a;
} output[outIdx] = a + (b - a) * frac;
output[outIdx] = count > 0 ? sum / count : 0;
inOffset = nextOffset;
} }
return output; return output;
} }
@@ -43,6 +42,16 @@ function floatToInt16Bytes(floatSamples) {
return bytes; return bytes;
} }
function concatUint8(chunks = [], totalLength = 0) {
const out = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
out.set(chunk, offset);
offset += chunk.length;
}
return out;
}
export default function VipAudioForwardingCard({ export default function VipAudioForwardingCard({
roster = [], roster = [],
ownRoverId = '', ownRoverId = '',
@@ -64,6 +73,8 @@ export default function VipAudioForwardingCard({
const mediaSourceRef = useRef(null); const mediaSourceRef = useRef(null);
const processorRef = useRef(null); const processorRef = useRef(null);
const sinkRef = useRef(null); const sinkRef = useRef(null);
const pendingPcmChunksRef = useRef([]);
const pendingPcmBytesRef = useRef(0);
const micActiveRef = useRef(false); const micActiveRef = useRef(false);
const activeRoverRef = useRef(''); const activeRoverRef = useRef('');
const singleRoverId = roster.length === 1 ? roster[0].id : ''; const singleRoverId = roster.length === 1 ? roster[0].id : '';
@@ -162,6 +173,8 @@ export default function VipAudioForwardingCard({
mediaSourceRef.current = null; mediaSourceRef.current = null;
sinkRef.current = null; sinkRef.current = null;
audioContextRef.current = null; audioContextRef.current = null;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
if (streamRef.current) { if (streamRef.current) {
try { try {
streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current.getTracks().forEach((track) => track.stop());
@@ -199,6 +212,7 @@ export default function VipAudioForwardingCard({
stream = await navigator.mediaDevices.getUserMedia({ stream = await navigator.mediaDevices.getUserMedia({
audio: { audio: {
channelCount: 1, channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: true, echoCancellation: true,
noiseSuppression: true, noiseSuppression: true,
autoGainControl: true, autoGainControl: true,
@@ -219,6 +233,8 @@ export default function VipAudioForwardingCard({
const sink = audioContext.createGain(); const sink = audioContext.createGain();
sink.gain.value = 0; sink.gain.value = 0;
sinkRef.current = sink; sinkRef.current = sink;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
micActiveRef.current = true; micActiveRef.current = true;
activeRoverRef.current = target; activeRoverRef.current = target;
@@ -226,10 +242,20 @@ export default function VipAudioForwardingCard({
if (!micActiveRef.current) return; if (!micActiveRef.current) return;
const input = event.inputBuffer?.getChannelData(0); const input = event.inputBuffer?.getChannelData(0);
if (!input || input.length === 0) return; if (!input || input.length === 0) return;
const downsampled = downsampleTo16k(input, audioContext.sampleRate); const resampled = resampleTo16k(input, audioContext.sampleRate);
if (!downsampled.length) return; if (!resampled.length) return;
const pcmBytes = floatToInt16Bytes(downsampled); const pcmBytes = floatToInt16Bytes(resampled);
sendMicChunk?.({ roverId: target, data: pcmBytes.buffer }); pendingPcmChunksRef.current.push(pcmBytes);
pendingPcmBytesRef.current += pcmBytes.length;
while (pendingPcmBytesRef.current >= MIC_PACKET_BYTES) {
const merged = concatUint8(pendingPcmChunksRef.current, pendingPcmBytesRef.current);
const packet = merged.slice(0, MIC_PACKET_BYTES);
const rest = merged.slice(MIC_PACKET_BYTES);
pendingPcmChunksRef.current = rest.length ? [rest] : [];
pendingPcmBytesRef.current = rest.length;
sendMicChunk?.({ roverId: target, data: packet });
}
}; };
source.connect(processor); source.connect(processor);
webui/src/context/SessionContext.jsx
+6
View File
@@ -153,7 +153,13 @@ export function SessionProvider({ children }) {
startMicForward: (roverId) => emitWithAck('audio:micStart', { roverId }), startMicForward: (roverId) => emitWithAck('audio:micStart', { roverId }),
stopMicForward: (roverId) => emitWithAck('audio:micStop', { roverId }), stopMicForward: (roverId) => emitWithAck('audio:micStop', { roverId }),
sendMicChunk: ({ roverId, dataBase64, data }) => { sendMicChunk: ({ roverId, dataBase64, data }) => {
if (!socket.connected) return false;
const ws = socket.io?.engine?.transport?.ws;
if (ws && typeof ws.bufferedAmount === 'number' && ws.bufferedAmount > 256 * 1024) {
return false;
}
socket.emit('audio:micChunk', { roverId, dataBase64, data }); socket.emit('audio:micChunk', { roverId, dataBase64, data });
return true;
}, },
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels), setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) => llmControl: (action, controls = {}) =>