legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-15 17:12:59 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

gwah

Browse Source
This commit is contained in:
legop3
2026-03-17 19:54:53 -04:00
parent 73ef6261ed
commit 5b8704ccb3
8 changed files with 183 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi/asound.conf
+2 -2
View File
@@ -12,8 +12,8 @@ pcm.dmixer {
rate 16000
channels 1
period_time 0
period_size 256
buffer_size 1024
period_size 1024
buffer_size 4096
}
}
pi/bin/audio-forward-listener.sh
+1 -5
View File
@@ -41,8 +41,6 @@ run_pipeline() {
-loglevel warning \
-fflags nobuffer \
-flags low_delay \
-max_delay 0 \
-reorder_queue_size 0 \
-analyzeduration 0 \
-probesize 32 \
-i "${AUDIO_FORWARD_URL}" \
@@ -57,9 +55,7 @@ run_pipeline() {
-t raw \
-f S16_LE \
-r 16000 \
-c 1 \
-B 40000 \
-F 10000
-c 1
local rc=$?
local -a statuses=("${PIPESTATUS[@]}")
LAST_FFMPEG_STATUS="${statuses[0]:-unknown}"
server/public/assets/index-CG8JYWgy.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-CXj15_94.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-CXj15_94.js"></script>
<script type="module" crossorigin src="/assets/index-CG8JYWgy.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head>
<body>
server/src/services/audioForwardService.js
+6
View File
@@ -450,6 +450,9 @@ function decodeMicChunk(payload = {}) {
if (Buffer.isBuffer(binary)) {
return binary;
}
if (binary && typeof binary === 'object' && binary.type === 'Buffer' && Array.isArray(binary.data)) {
return Buffer.from(binary.data);
}
if (binary instanceof Uint8Array) {
return Buffer.from(binary.buffer, binary.byteOffset, binary.byteLength);
}
@@ -474,6 +477,9 @@ function pushMicChunk(roverId, ownerSocketId, payload = {}) {
if (!bytes.length) {
throw new Error('Mic chunk missing');
}
if (bytes.length % 2 !== 0) {
throw new Error('Mic chunk has invalid PCM byte length');
}
if (bytes.length > 64 * 1024) {
throw new Error('Mic chunk too large');
}
webui/src/components/vip/VipAudioForwardingCard.jsx
+42 -16
View File
@@ -9,25 +9,24 @@ import {
import { useControlSystem } from '../../controls/index.js';
const TARGET_SAMPLE_RATE = 16000;
const MIC_PACKET_MS = 40;
const MIC_PACKET_BYTES = (TARGET_SAMPLE_RATE * 2 * MIC_PACKET_MS) / 1000; // s16le mono
function downsampleTo16k(input, sampleRate) {
function resampleTo16k(input, sampleRate) {
if (!input || !input.length) return new Float32Array(0);
if (sampleRate === TARGET_SAMPLE_RATE) return input;
if (!Number.isFinite(sampleRate) || sampleRate < TARGET_SAMPLE_RATE) return input;
if (!Number.isFinite(sampleRate) || sampleRate <= 0) return input;
const ratio = sampleRate / TARGET_SAMPLE_RATE;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const output = new Float32Array(outputLength);
let inOffset = 0;
for (let outIdx = 0; outIdx < outputLength; outIdx += 1) {
const nextOffset = Math.min(input.length, Math.round((outIdx + 1) * ratio));
let sum = 0;
let count = 0;
for (let i = inOffset; i < nextOffset; i += 1) {
sum += input[i];
count += 1;
}
output[outIdx] = count > 0 ? sum / count : 0;
inOffset = nextOffset;
const src = outIdx * ratio;
const srcFloor = Math.floor(src);
const srcCeil = Math.min(input.length - 1, srcFloor + 1);
const frac = src - srcFloor;
const a = input[srcFloor] ?? 0;
const b = input[srcCeil] ?? a;
output[outIdx] = a + (b - a) * frac;
}
return output;
}
@@ -43,6 +42,16 @@ function floatToInt16Bytes(floatSamples) {
return bytes;
}
function concatUint8(chunks = [], totalLength = 0) {
const out = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
out.set(chunk, offset);
offset += chunk.length;
}
return out;
}
export default function VipAudioForwardingCard({
roster = [],
ownRoverId = '',
@@ -64,6 +73,8 @@ export default function VipAudioForwardingCard({
const mediaSourceRef = useRef(null);
const processorRef = useRef(null);
const sinkRef = useRef(null);
const pendingPcmChunksRef = useRef([]);
const pendingPcmBytesRef = useRef(0);
const micActiveRef = useRef(false);
const activeRoverRef = useRef('');
const singleRoverId = roster.length === 1 ? roster[0].id : '';
@@ -162,6 +173,8 @@ export default function VipAudioForwardingCard({
mediaSourceRef.current = null;
sinkRef.current = null;
audioContextRef.current = null;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
if (streamRef.current) {
try {
streamRef.current.getTracks().forEach((track) => track.stop());
@@ -199,6 +212,7 @@ export default function VipAudioForwardingCard({
stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
@@ -219,6 +233,8 @@ export default function VipAudioForwardingCard({
const sink = audioContext.createGain();
sink.gain.value = 0;
sinkRef.current = sink;
pendingPcmChunksRef.current = [];
pendingPcmBytesRef.current = 0;
micActiveRef.current = true;
activeRoverRef.current = target;
@@ -226,10 +242,20 @@ export default function VipAudioForwardingCard({
if (!micActiveRef.current) return;
const input = event.inputBuffer?.getChannelData(0);
if (!input || input.length === 0) return;
const downsampled = downsampleTo16k(input, audioContext.sampleRate);
if (!downsampled.length) return;
const pcmBytes = floatToInt16Bytes(downsampled);
sendMicChunk?.({ roverId: target, data: pcmBytes.buffer });
const resampled = resampleTo16k(input, audioContext.sampleRate);
if (!resampled.length) return;
const pcmBytes = floatToInt16Bytes(resampled);
pendingPcmChunksRef.current.push(pcmBytes);
pendingPcmBytesRef.current += pcmBytes.length;
while (pendingPcmBytesRef.current >= MIC_PACKET_BYTES) {
const merged = concatUint8(pendingPcmChunksRef.current, pendingPcmBytesRef.current);
const packet = merged.slice(0, MIC_PACKET_BYTES);
const rest = merged.slice(MIC_PACKET_BYTES);
pendingPcmChunksRef.current = rest.length ? [rest] : [];
pendingPcmBytesRef.current = rest.length;
sendMicChunk?.({ roverId: target, data: packet });
}
};
source.connect(processor);
webui/src/context/SessionContext.jsx
+6
View File
@@ -153,7 +153,13 @@ export function SessionProvider({ children }) {
startMicForward: (roverId) => emitWithAck('audio:micStart', { roverId }),
stopMicForward: (roverId) => emitWithAck('audio:micStop', { roverId }),
sendMicChunk: ({ roverId, dataBase64, data }) => {
if (!socket.connected) return false;
const ws = socket.io?.engine?.transport?.ws;
if (ws && typeof ws.bufferedAmount === 'number' && ws.bufferedAmount > 256 * 1024) {
return false;
}
socket.emit('audio:micChunk', { roverId, dataBase64, data });
return true;
},
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) =>