legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-15 17:12:59 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

wewa

Browse Source
This commit is contained in:
legop3
2026-03-17 19:35:36 -04:00
parent 618fd3e4de
commit 93204ec1f1
12 changed files with 529 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/public/assets/index-CtTWAec1.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-D4q08ieD.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-D4q08ieD.js"></script>
<script type="module" crossorigin src="/assets/index-CtTWAec1.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-WqYsCIRI.css">
</head>
<body>
server/src/services/audioForwardService.js
+188 -7
View File
@@ -145,7 +145,7 @@ function resolveForwardUrl(roverId) {
function spawnProcess(roverId, tag, args, options = {}) {
const proc = spawn(ffmpegBin, args, {
stdio: ['ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'],
stdio: [options.captureStdin ? 'pipe' : 'ignore', options.captureStdout ? 'pipe' : 'ignore', 'pipe'],
});
proc.stderr?.on('data', (chunk) => {
const text = String(chunk || '').trim();
@@ -258,6 +258,30 @@ function buildUploadWriterArgs(filePath) {
];
}
function buildMicWriterArgs() {
return [
'-hide_banner',
'-loglevel',
'warning',
'-fflags',
'nobuffer',
'-flags',
'low_delay',
'-i',
'pipe:0',
'-vn',
'-af',
'aresample=16000',
'-f',
's16le',
'-ac',
'1',
'-ar',
'16000',
'pipe:1',
];
}
function attachWriterPipe(worker, proc) {
const writer = fs.createWriteStream(worker.fifoPath, { flags: 'w' });
writer.on('error', (err) => {
@@ -292,6 +316,18 @@ function cleanupUploadFile(worker) {
function stopContentWriter(worker) {
if (!worker?.contentProc) return;
if (worker.micIdleTimer) {
clearTimeout(worker.micIdleTimer);
worker.micIdleTimer = null;
}
worker.micLastChunkAt = 0;
if (worker.contentProc.stdin && !worker.contentProc.stdin.destroyed) {
try {
worker.contentProc.stdin.destroy();
} catch {
// noop
}
}
stopProc(worker.contentProc);
worker.contentProc = null;
worker.contentKind = null;
@@ -358,6 +394,100 @@ function startUploadWriter(roverId, filePath) {
});
}
function scheduleMicIdleTimeout(roverId) {
const worker = workers.get(roverId);
if (!worker || worker.contentKind !== 'mic') return;
if (worker.micIdleTimer) {
clearTimeout(worker.micIdleTimer);
}
worker.micIdleTimer = setTimeout(() => {
const current = workers.get(roverId);
if (!current || current.contentKind !== 'mic') return;
const staleForMs = Date.now() - (current.micLastChunkAt || 0);
if (staleForMs < 2500) return;
logger.info('Stopping mic writer due to idle chunk timeout', { roverId, staleForMs });
startSilenceWriter(roverId);
}, 3000);
}
function startMicWriter(roverId, ownerSocketId = null) {
const worker = workers.get(roverId);
if (!worker || worker.stopping) return;
if (
worker.contentKind === 'mic' &&
worker.contentProc &&
worker.activeOwnerSocketId === ownerSocketId
) {
return;
}
stopContentWriter(worker);
cleanupUploadFile(worker);
const proc = spawnProcess(roverId, 'mic-writer', buildMicWriterArgs(), {
captureStdout: true,
captureStdin: true,
});
worker.contentProc = proc;
worker.contentKind = 'mic';
worker.activeOwnerSocketId = ownerSocketId;
worker.micLastChunkAt = Date.now();
scheduleMicIdleTimeout(roverId);
const seq = ++worker.writerSeq;
attachWriterPipe(worker, proc);
proc.stdin?.on('error', (err) => {
const code = err?.code || 'unknown';
if (code !== 'EPIPE') {
logger.warn('mic writer stdin error', { roverId, code, message: err?.message || String(err) });
}
});
setState(roverId, { state: 'playing', source: 'mic', error: null, startedAt: Date.now() });
proc.on('exit', (code, signal) => {
const current = workers.get(roverId);
if (!current || current.stopping) return;
if (current.writerSeq !== seq || current.contentProc !== proc) return;
current.contentProc = null;
current.contentKind = null;
current.activeOwnerSocketId = null;
if (current.micIdleTimer) {
clearTimeout(current.micIdleTimer);
current.micIdleTimer = null;
}
current.micLastChunkAt = 0;
if (code != null && code !== 0 && signal !== 'SIGTERM') {
setState(roverId, { state: 'error', source: 'mic', error: `mic writer exited code=${code} signal=${signal || 'none'}` });
}
startSilenceWriter(roverId);
});
}
function pushMicChunk(roverId, ownerSocketId, dataBase64) {
const worker = workers.get(roverId);
if (!worker) {
throw new Error('Audio forward worker unavailable');
}
if (worker.contentKind !== 'mic' || !worker.contentProc || worker.activeOwnerSocketId !== ownerSocketId) {
throw new Error('Mic forwarding is not active');
}
const encoded = typeof dataBase64 === 'string' ? dataBase64.trim() : '';
if (!encoded) {
throw new Error('Mic chunk missing');
}
const bytes = Buffer.from(encoded, 'base64');
if (!bytes.length) {
throw new Error('Mic chunk decode failed');
}
if (worker.contentProc.stdin?.writable !== true) {
throw new Error('Mic writer input is not writable');
}
worker.micLastChunkAt = Date.now();
worker.contentProc.stdin.write(bytes);
scheduleMicIdleTimeout(roverId);
}
function ensureWorker(roverId) {
ensureServiceEnabled();
if (!roverId) {
@@ -388,7 +518,10 @@ function ensureWorker(roverId) {
publisherProc: publisher,
contentProc: null,
contentKind: null,
activeOwnerSocketId: null,
activeUploadPath: null,
micLastChunkAt: 0,
micIdleTimer: null,
writerSeq: 0,
stopping: false,
};
@@ -490,31 +623,32 @@ roverManager.managerEvents.on('rover', ({ roverId, action } = {}) => {
}
});
function stopOwnedUploadIfUnauthorized(roverId, ownerSocketId, reason = 'driver_change') {
function stopOwnedAudioIfUnauthorized(roverId, ownerSocketId, reason = 'driver_change') {
if (!roverId || !ownerSocketId) return;
const worker = workers.get(roverId);
if (!worker || worker.contentKind !== 'upload') return;
if (!worker) return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic') return;
if (worker.activeOwnerSocketId !== ownerSocketId) return;
const ownerSocket = io.sockets.sockets.get(ownerSocketId);
const ownerIsDriver = ownerSocket ? roverManager.isDriver(roverId, ownerSocket) : false;
const ownerCanDrive = ownerSocket ? turnService.canDrive(roverId, ownerSocket) : false;
if (ownerIsDriver && ownerCanDrive) return;
logger.info('Stopping upload due to ownership/driver change', { roverId, ownerSocketId, reason });
logger.info('Stopping audio forward due to ownership/driver change', { roverId, ownerSocketId, reason, source: worker.contentKind });
startSilenceWriter(roverId);
}
roverManager.managerEvents.on('driver', ({ socketId, roverId, action } = {}) => {
if (!socketId || !roverId) return;
if (action === 'remove' || action === 'add') {
stopOwnedUploadIfUnauthorized(roverId, socketId, action);
stopOwnedAudioIfUnauthorized(roverId, socketId, action);
}
});
turnService.turnEvents.on('activeDriver', ({ roverId } = {}) => {
if (!roverId) return;
const worker = workers.get(roverId);
if (!worker || worker.contentKind !== 'upload') return;
stopOwnedUploadIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
if (!worker || (worker.contentKind !== 'upload' && worker.contentKind !== 'mic')) return;
stopOwnedAudioIfUnauthorized(roverId, worker.activeOwnerSocketId, 'turn_change');
});
io.on('connection', (socket) => {
@@ -540,6 +674,53 @@ io.on('connection', (socket) => {
cb({ error: err.message });
}
});
socket.on('audio:micStart', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
ensureWorker(normalized);
startMicWriter(normalized, socket.id);
cb({ success: true, roverId: normalized });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('audio:micChunk', (payload = {}, cb) => {
try {
const normalized = String(payload?.roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
pushMicChunk(normalized, socket.id, payload?.dataBase64);
if (typeof cb === 'function') cb({ success: true });
} catch (err) {
if (typeof cb === 'function') cb({ error: err.message });
}
});
socket.on('audio:micStop', ({ roverId } = {}, cb = () => {}) => {
try {
const normalized = String(roverId || '').trim();
ensureAudioForwardPermission(socket, normalized);
const worker = workers.get(normalized);
if (worker && worker.contentKind === 'mic' && worker.activeOwnerSocketId !== socket.id) {
throw new Error('Mic forwarding is owned by another session');
}
stopPlayback(normalized);
cb({ success: true, roverId: normalized });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('disconnect', () => {
workers.forEach((worker, roverId) => {
if (!worker || worker.activeOwnerSocketId !== socket.id) return;
if (worker.contentKind !== 'upload' && worker.contentKind !== 'mic') return;
logger.info('Stopping owned audio forward due to socket disconnect', { roverId, socketId: socket.id, source: worker.contentKind });
startSilenceWriter(roverId);
});
});
});
module.exports = {
webui/src/components/KeymapSettings.jsx
+1
View File
@@ -24,6 +24,7 @@ const KEY_ACTIONS = [
{ id: 'cameraDown', label: 'Camera Down', group: 'Camera' },
{ id: 'nightVisionToggle', label: 'Toggle Night Vision', group: 'Camera' },
{ id: 'hornHonk', label: 'Horn (Hold)', group: 'Audio' },
{ id: 'micPtt', label: 'Mic Push To Talk', group: 'Audio' },
{ id: 'driveMacro', label: 'Drive Macro', group: 'Macros' },
{ id: 'dockMacro', label: 'Dock Macro', group: 'Macros' },
{ id: 'chatFocus', label: 'Toggle Chat', group: 'Chat' },
webui/src/components/VipPanel.jsx
+13 -1
View File
@@ -7,7 +7,16 @@ import VipVerificationCard from './vip/VipVerificationCard.jsx';
import VipIdentityCard from './vip/VipIdentityCard.jsx';
export default function VipPanel() {
const { session, identifySession, requestVerification, playUploadedAudio, stopUploadedAudio } = useSession();
const {
session,
identifySession,
requestVerification,
playUploadedAudio,
stopUploadedAudio,
startMicForward,
stopMicForward,
sendMicChunk,
} = useSession();
const { value: identity, save: saveIdentity } = useSettingsNamespace('identity', { cookieUserId: '' });
const { value: profile } = useSettingsNamespace('profile', { nickname: '' });
@@ -45,6 +54,9 @@ export default function VipPanel() {
audioForwardByRover={session?.audioForward || {}}
playUploadedAudio={playUploadedAudio}
stopUploadedAudio={stopUploadedAudio}
startMicForward={startMicForward}
stopMicForward={stopMicForward}
sendMicChunk={sendMicChunk}
/>
) : (
<VipVerificationCard
webui/src/components/vip/VipAudioForwardingCard.jsx
+158 -1
View File
@@ -1,4 +1,4 @@
import { useMemo, useState } from 'react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import {
MAX_UPLOAD_BYTES,
bytesToBase64,
@@ -6,6 +6,7 @@ import {
flowWrapClass,
innerFlowClass,
} from './constants.js';
import { useControlSystem } from '../../controls/index.js';
export default function VipAudioForwardingCard({
roster = [],
@@ -13,12 +14,23 @@ export default function VipAudioForwardingCard({
audioForwardByRover = {},
playUploadedAudio,
stopUploadedAudio,
startMicForward,
stopMicForward,
sendMicChunk,
}) {
const { state: controlState } = useControlSystem();
const [selectedUpload, setSelectedUpload] = useState(null);
const [working, setWorking] = useState(false);
const [openMicEnabled, setOpenMicEnabled] = useState(false);
const [micState, setMicState] = useState('idle');
const [message, setMessage] = useState('');
const recorderRef = useRef(null);
const streamRef = useRef(null);
const micActiveRef = useRef(false);
const activeRoverRef = useRef('');
const singleRoverId = roster.length === 1 ? roster[0].id : '';
const targetRoverId = String(singleRoverId || ownRoverId || '').trim();
const pttActive = Boolean(controlState?.mic?.pttActive);
const selectedForwardState = useMemo(
() => (targetRoverId ? audioForwardByRover?.[targetRoverId] || null : null),
[audioForwardByRover, targetRoverId],
@@ -75,6 +87,137 @@ export default function VipAudioForwardingCard({
}
};
const stopMicCapture = useCallback(
async (roverId) => {
const target = String(roverId || activeRoverRef.current || '').trim();
micActiveRef.current = false;
setMicState('idle');
try {
const recorder = recorderRef.current;
if (recorder && recorder.state !== 'inactive') {
recorder.stop();
}
} catch {
// noop
}
recorderRef.current = null;
if (streamRef.current) {
try {
streamRef.current.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
}
streamRef.current = null;
if (target) {
try {
await stopMicForward?.(target);
} catch {
// noop
}
}
activeRoverRef.current = '';
},
[stopMicForward],
);
const startMicCapture = useCallback(
async (roverId) => {
const target = String(roverId || '').trim();
if (!target) {
throw new Error('Take control of a rover first.');
}
if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('Microphone capture is not supported in this browser.');
}
if (typeof MediaRecorder === 'undefined') {
throw new Error('MediaRecorder is not supported in this browser.');
}
await stopMicCapture(target);
setMicState('starting');
let stream = null;
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
streamRef.current = stream;
await startMicForward?.(target);
const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
? 'audio/webm;codecs=opus'
: 'audio/webm';
const recorder = new MediaRecorder(stream, { mimeType, audioBitsPerSecond: 64000 });
recorderRef.current = recorder;
micActiveRef.current = true;
activeRoverRef.current = target;
recorder.ondataavailable = async (event) => {
try {
if (!micActiveRef.current || !event.data || event.data.size <= 0) return;
const buffer = await event.data.arrayBuffer();
const base64 = bytesToBase64(new Uint8Array(buffer));
sendMicChunk?.({ roverId: target, dataBase64: base64 });
} catch {
// noop
}
};
recorder.onstop = () => {
if (!micActiveRef.current) return;
micActiveRef.current = false;
setMicState('idle');
};
recorder.start(120);
setMicState('live');
} catch (err) {
if (stream) {
try {
stream.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
}
streamRef.current = null;
throw err;
}
},
[sendMicChunk, startMicForward, stopMicCapture],
);
useEffect(() => {
const desiredActive = Boolean(openMicEnabled || pttActive);
const roverId = targetRoverId;
let cancelled = false;
async function syncMicState() {
if (!roverId || !desiredActive) {
await stopMicCapture(activeRoverRef.current || roverId);
return;
}
if (micActiveRef.current && activeRoverRef.current === roverId) return;
try {
await startMicCapture(roverId);
} catch (err) {
if (!cancelled) {
setMicState('error');
setMessage(err?.message || 'Failed to start microphone forwarding.');
}
}
}
syncMicState();
return () => {
cancelled = true;
};
}, [openMicEnabled, pttActive, startMicCapture, stopMicCapture, targetRoverId]);
useEffect(
() => () => {
stopMicCapture(activeRoverRef.current);
},
[stopMicCapture],
);
return (
<section className={`surface ${flowWrapClass}`}>
<div className={innerFlowClass}>
@@ -102,6 +245,20 @@ export default function VipAudioForwardingCard({
Stop
</button>
</div>
<div className="surface-muted mx-auto flex w-full max-w-sm flex-col gap-0.5 p-0.5 text-xs text-slate-300 text-center">
<p className="text-slate-200">Microphone Forwarding</p>
<label className="flex items-center justify-center gap-0.5">
<input
type="checkbox"
checked={openMicEnabled}
disabled={!targetRoverId}
onChange={(event) => setOpenMicEnabled(Boolean(event.target.checked))}
/>
<span>Open mic</span>
</label>
<p className="text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'v'} (hold)</p>
<p className="text-slate-400">mic: {micState}</p>
</div>
{selectedForwardState ? (
<div className="surface-muted mx-auto w-full max-w-sm text-xs text-slate-300 text-center">
state: {selectedForwardState.state || 'idle'}
webui/src/context/SessionContext.jsx
+9 -1
View File
@@ -28,6 +28,9 @@ const SessionContext = createContext({
rebootServer: async () => {},
playUploadedAudio: async () => {},
stopUploadedAudio: async () => {},
startMicForward: async () => {},
stopMicForward: async () => {},
sendMicChunk: async () => {},
setAudioLevels: async () => {},
llmControl: async () => {},
});
@@ -147,6 +150,11 @@ export function SessionProvider({ children }) {
playUploadedAudio: ({ roverId, name, mime, dataBase64 }) =>
emitWithAck('audio:uploadPlay', { roverId, name, mime, dataBase64 }),
stopUploadedAudio: (roverId) => emitWithAck('audio:uploadStop', { roverId }),
startMicForward: (roverId) => emitWithAck('audio:micStart', { roverId }),
stopMicForward: (roverId) => emitWithAck('audio:micStop', { roverId }),
sendMicChunk: ({ roverId, dataBase64 }) => {
socket.emit('audio:micChunk', { roverId, dataBase64 });
},
setAudioLevels: (levels = {}) => emitWithAck('audioLevels:set', levels),
llmControl: (action, controls = {}) =>
emitWithAck('llm:control', { controls: { action, ...controls } }),
@@ -156,7 +164,7 @@ export function SessionProvider({ children }) {
{ ...alert, receivedAt: Date.now(), id: alert.id || Math.random().toString(36).slice(2) },
]),
}),
[emitWithAck],
[emitWithAck, socket],
);
const value = useMemo(
webui/src/controls/ControlContext.jsx
+6
View File
@@ -452,6 +452,10 @@ export function ControlSystemProvider({ children }) {
dispatch({ type: 'control/register-input-state', payload: { source, state: data } });
}, []);
const setMicPttActive = useCallback((active) => {
dispatch({ type: 'control/set-mic-ptt', payload: Boolean(active) });
}, []);
const contextValue = useMemo(
() => ({
state,
@@ -478,6 +482,7 @@ export function ControlSystemProvider({ children }) {
sendSong,
startHorn,
stopHorn,
setMicPttActive,
},
}),
[
@@ -503,6 +508,7 @@ export function ControlSystemProvider({ children }) {
sendSong,
startHorn,
stopHorn,
setMicPttActive,
],
);
webui/src/controls/constants.js
+1
View File
@@ -48,6 +48,7 @@ export const DEFAULT_KEYMAP = {
cameraDown: ['j'],
nightVisionToggle: ['e'],
hornHonk: ['h'],
micPtt: ['v'],
driveMacro: ['f'],
dockMacro: ['g'],
chatFocus: ['enter'],
webui/src/controls/controlReducer.js
+17
View File
@@ -35,6 +35,12 @@ function createHornState() {
};
}
function createMicState() {
return {
pttActive: false,
};
}
export const initialControlState = {
roverId: null,
mode: 'drive',
@@ -43,6 +49,7 @@ export const initialControlState = {
camera: createCameraState(),
song: createSongState(),
horn: createHornState(),
mic: createMicState(),
lastControlIntentAt: 0,
macros: DEFAULT_MACROS,
keymap: DEFAULT_KEYMAP,
@@ -59,6 +66,7 @@ export function controlReducer(state, action) {
aux: action.payload ? state.aux : createAuxState(),
song: action.payload ? state.song : createSongState(),
horn: action.payload ? state.horn : createHornState(),
mic: action.payload ? state.mic : createMicState(),
lastControlIntentAt: action.payload ? state.lastControlIntentAt : 0,
};
case 'control/set-mode':
@@ -136,6 +144,7 @@ export function controlReducer(state, action) {
aux: createAuxState(),
song: createSongState(),
horn: createHornState(),
mic: createMicState(),
lastControlIntentAt: 0,
};
case 'control/set-horn-active':
@@ -168,6 +177,14 @@ export function controlReducer(state, action) {
note: action.payload ?? SONG_DEFAULT_NOTE,
},
};
case 'control/set-mic-ptt':
return {
...state,
mic: {
...(state.mic || createMicState()),
pttActive: Boolean(action.payload),
},
};
default:
return state;
}
webui/src/controls/inputs/KeyboardInputManager.jsx
+10 -1
View File
@@ -129,6 +129,7 @@ export default function KeyboardInputManager() {
toggleNightVision,
startHorn,
stopHorn,
setMicPttActive,
setSongNote,
sendSong,
},
@@ -307,9 +308,10 @@ export default function KeyboardInputManager() {
hornActiveRef.current = false;
stopHorn();
}
setMicPttActive(false);
stopAllMotion();
registerInputState(SOURCE, { keys: [], vector: ZERO_VECTOR, aux: ZERO_AUX });
}, [registerInputState, stopAllMotion, stopHorn, stopServoLoop, stopSongLoop]);
}, [registerInputState, setMicPttActive, stopAllMotion, stopHorn, stopServoLoop, stopSongLoop]);
const triggerHomeAssistantCycle = useCallback(
(targetState) => {
@@ -378,6 +380,8 @@ export default function KeyboardInputManager() {
const started = startHorn();
hornActiveRef.current = Boolean(started);
}
} else if (newlyPressed.some((token) => keymap.micPtt?.has(token))) {
setMicPttActive(true);
} else if (newlyPressed.some((token) => keymap.homeAssistantOn?.has(token))) {
triggerHomeAssistantCycle('on');
} else if (newlyPressed.some((token) => keymap.homeAssistantOff?.has(token))) {
@@ -398,6 +402,9 @@ export default function KeyboardInputManager() {
hornActiveRef.current = false;
stopHorn();
}
if (!bindingActive(keymap.micPtt, activeTokensRef.current)) {
setMicPttActive(false);
}
ensureServoLoop();
ensureSongLoop();
driveFromKeys();
@@ -427,8 +434,10 @@ export default function KeyboardInputManager() {
keymap.dockMacro,
keymap.driveMacro,
keymap.hornHonk,
keymap.micPtt,
resetAll,
runMacro,
setMicPttActive,
setMode,
stopAllMotion,
stopSongLoop,