legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 01:21:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

gordon

Browse Source
This commit is contained in:
legop3
2026-04-06 01:17:25 -04:00
parent 985e63bab2
commit 5e428b278a
7 changed files with 686 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/public/assets/index-dMtKgicC.js
-125
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-oXFJbPgP.js
+125
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-dMtKgicC.js"></script> <script type="module" crossorigin src="/assets/index-oXFJbPgP.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BPze86Ff.css"> <link rel="stylesheet" crossorigin href="/assets/index-BPze86Ff.css">
</head> </head>
<body> <body>
server/src/services/roomHumanDetectionService.js
+258 -53
View File
@@ -2,6 +2,7 @@ const path = require('path');
const { spawn } = require('child_process'); const { spawn } = require('child_process');
const readline = require('readline'); const readline = require('readline');
const logger = require('../globals/logger').child('roomHumanDetection'); const logger = require('../globals/logger').child('roomHumanDetection');
const io = require('../globals/io');
const { loadConfig } = require('../helpers/configLoader'); const { loadConfig } = require('../helpers/configLoader');
const { roomCameraStreamEvents } = require('./roomCameraSnapshotService'); const { roomCameraStreamEvents } = require('./roomCameraSnapshotService');
const roverManager = require('./roverManager'); const roverManager = require('./roverManager');
@@ -9,43 +10,40 @@ const { issueCommand } = require('./commandService');
const { publishEvent } = require('./eventBus'); const { publishEvent } = require('./eventBus');
const { sendAlert } = require('./alertService'); const { sendAlert } = require('./alertService');
const { getMode, MODES } = require('./modeManager'); const { getMode, MODES } = require('./modeManager');
const { isAdmin } = require('./roleService');
const config = loadConfig(); const config = loadConfig();
const visionConfig = config.vision?.humanDetection || {}; const visionConfig = config.vision?.humanDetection || {};
const enabled = Boolean(visionConfig.enabled);
if (!enabled) { const runtime = {
logger.info('Room human detection disabled via config'); enabled: Boolean(visionConfig.enabled),
return; confidenceThreshold: Number.isFinite(Number(visionConfig.confidenceThreshold))
} ? Number(visionConfig.confidenceThreshold)
: 0.55,
ttsDelayMs: Number.isFinite(Number(visionConfig.ttsDelayMs)) ? Number(visionConfig.ttsDelayMs) : 30000,
discordDelayMs: Number.isFinite(Number(visionConfig.discordDelayMs)) ? Number(visionConfig.discordDelayMs) : 60000,
clearWindowMs: Number.isFinite(Number(visionConfig.clearWindowMs)) ? Number(visionConfig.clearWindowMs) : 3000,
cooldownMs: Number.isFinite(Number(visionConfig.cooldownMs)) ? Number(visionConfig.cooldownMs) : 15 * 60 * 1000,
maxInferenceFpsPerCamera: Number.isFinite(Number(visionConfig.maxInferenceFpsPerCamera))
? Math.max(0.25, Number(visionConfig.maxInferenceFpsPerCamera))
: 3,
};
const CONFIDENCE_THRESHOLD = Number.isFinite(Number(visionConfig.confidenceThreshold))
? Number(visionConfig.confidenceThreshold)
: 0.55;
const TTS_DELAY_MS = Number.isFinite(Number(visionConfig.ttsDelayMs))
? Number(visionConfig.ttsDelayMs)
: 30000;
const DISCORD_DELAY_MS = Number.isFinite(Number(visionConfig.discordDelayMs))
? Number(visionConfig.discordDelayMs)
: 60000;
const CLEAR_WINDOW_MS = Number.isFinite(Number(visionConfig.clearWindowMs))
? Number(visionConfig.clearWindowMs)
: 3000;
const COOLDOWN_MS = Number.isFinite(Number(visionConfig.cooldownMs))
? Number(visionConfig.cooldownMs)
: 15 * 60 * 1000;
const MAX_INFERENCE_FPS = Number.isFinite(Number(visionConfig.maxInferenceFpsPerCamera))
? Math.max(0.25, Number(visionConfig.maxInferenceFpsPerCamera))
: 3;
const TTS_TEXT = 'human detected in room, alerting soon'; const TTS_TEXT = 'human detected in room, alerting soon';
const workerScript = path.join(__dirname, '..', '..', 'scripts', 'human_detector_worker.py'); const workerScript = path.join(__dirname, '..', '..', 'scripts', 'human_detector_worker.py');
const pythonCandidates = [process.env.VISION_PYTHON, '/opt/multiroomba-vision/.venv/bin/python3', 'python3'].filter(Boolean); const pythonCandidates = [
const inferenceIntervalMs = Math.max(100, Math.round(1000 / MAX_INFERENCE_FPS)); process.env.VISION_PYTHON,
'/opt/multiroomba-vision/.venv/bin/python3',
'python3',
].filter(Boolean);
const cameraState = new Map(); // cameraId -> { inflight,lastInferAt,lastResultAt,lastPositiveAt,lastConfidence,error } const cameraState = new Map(); // cameraId -> { inflight,lastInferAt,lastResultAt,lastPositiveAt,lastConfidence,error }
const history = []; // up to 80 recent events
let worker = null; let worker = null;
let workerReady = false; let workerReady = false;
let workerPython = null;
let workerRestartCount = 0;
let reqSeq = 1; let reqSeq = 1;
let lastAnyPositiveAt = null; let lastAnyPositiveAt = null;
@@ -55,6 +53,17 @@ let discordSentThisEpisode = false;
let latestPositiveFrame = null; // { cameraId, confidence, ts, buffer } let latestPositiveFrame = null; // { cameraId, confidence, ts, buffer }
let cooldownUntil = 0; let cooldownUntil = 0;
let hasClearedSinceLastDiscord = true; let hasClearedSinceLastDiscord = true;
let lastDiscordAlertAt = null;
let lastDiscordAlertMeta = null;
function getInferenceIntervalMs() {
return Math.max(100, Math.round(1000 / Math.max(0.25, Number(runtime.maxInferenceFpsPerCamera) || 3)));
}
function pushHistory(type, detail = {}) {
history.push({ ts: Date.now(), type, detail });
while (history.length > 80) history.shift();
}
function isDetectionActiveMode() { function isDetectionActiveMode() {
const mode = getMode(); const mode = getMode();
@@ -75,7 +84,7 @@ function ensureCameraState(cameraId) {
return cameraState.get(cameraId); return cameraState.get(cameraId);
} }
function markCleared(now) { function markCleared(now, reason = 'clear_window') {
if (episodeStartAt == null) return; if (episodeStartAt == null) return;
episodeStartAt = null; episodeStartAt = null;
ttsSentThisEpisode = false; ttsSentThisEpisode = false;
@@ -83,7 +92,8 @@ function markCleared(now) {
lastAnyPositiveAt = null; lastAnyPositiveAt = null;
latestPositiveFrame = null; latestPositiveFrame = null;
hasClearedSinceLastDiscord = true; hasClearedSinceLastDiscord = true;
logger.info('Human detection episode cleared', { now }); pushHistory('episode.cleared', { reason });
logger.info('Human detection episode cleared', { now, reason });
} }
function sendTtsToNonPrivateRovers() { function sendTtsToNonPrivateRovers() {
@@ -107,7 +117,10 @@ function sendTtsToNonPrivateRovers() {
sendAlert({ sendAlert({
color: '#f0b651', color: '#f0b651',
title: 'Human Detection', title: 'Human Detection',
message: sent > 0 ? `Person detected; announced on ${sent} rover(s).` : 'Person detected; no non-private rover available.', message:
sent > 0
? `Person detected; announced on ${sent} rover(s).`
: 'Person detected; no non-private rover available.',
}); });
publishEvent({ publishEvent({
source: 'roomHumanDetection', source: 'roomHumanDetection',
@@ -118,11 +131,12 @@ function sendTtsToNonPrivateRovers() {
ts: Date.now(), ts: Date.now(),
}, },
}); });
pushHistory('tts.sent', { roverCount: sent });
} }
function sendDiscordDetectionAlert(now) { function sendDiscordDetectionAlert(now, options = {}) {
const payload = { const payload = {
message: 'Human detected in room cameras.', message: options.message || 'Human detected in room cameras.',
detectedAt: now, detectedAt: now,
confidence: latestPositiveFrame?.confidence || null, confidence: latestPositiveFrame?.confidence || null,
cameraId: latestPositiveFrame?.cameraId || null, cameraId: latestPositiveFrame?.cameraId || null,
@@ -138,31 +152,108 @@ function sendDiscordDetectionAlert(now) {
title: 'Human Detection', title: 'Human Detection',
message: 'Human presence persisted; Discord alert sent.', message: 'Human presence persisted; Discord alert sent.',
}); });
lastDiscordAlertAt = now;
lastDiscordAlertMeta = {
cameraId: payload.cameraId,
confidence: payload.confidence,
detectedAt: payload.detectedAt,
};
pushHistory('discord.sent', {
cameraId: payload.cameraId,
confidence: payload.confidence,
});
}
function buildState(now = Date.now()) {
const mode = getMode();
const modeActive = isDetectionActiveMode();
const humanPresent = episodeStartAt != null;
const elapsed = humanPresent ? Math.max(0, now - episodeStartAt) : 0;
const timeToTtsMs = humanPresent && !ttsSentThisEpisode ? Math.max(0, runtime.ttsDelayMs - elapsed) : 0;
const timeToDiscordMs = humanPresent && !discordSentThisEpisode ? Math.max(0, runtime.discordDelayMs - elapsed) : 0;
const cooldownRemainingMs = Math.max(0, cooldownUntil - now);
const cameras = Array.from(cameraState.entries()).map(([cameraId, state]) => ({
cameraId,
inflight: Boolean(state.inflight),
lastInferAt: state.lastInferAt || null,
lastResultAt: state.lastResultAt || null,
lastPositiveAt: state.lastPositiveAt || null,
lastConfidence: state.lastConfidence || 0,
error: state.error || null,
}));
cameras.sort((a, b) => String(a.cameraId).localeCompare(String(b.cameraId)));
return {
enabled: runtime.enabled,
mode,
modeActive,
workerReady,
workerRunning: Boolean(worker && !worker.killed),
workerPython,
workerScript,
workerRestartCount,
config: { ...runtime },
episode: {
humanPresent,
startAt: episodeStartAt,
lastAnyPositiveAt,
ttsSentThisEpisode,
discordSentThisEpisode,
timeToTtsMs,
timeToDiscordMs,
cooldownUntil,
cooldownRemainingMs,
hasClearedSinceLastDiscord,
},
latestPositive: latestPositiveFrame
? {
cameraId: latestPositiveFrame.cameraId,
confidence: latestPositiveFrame.confidence,
ts: latestPositiveFrame.ts,
}
: null,
lastDiscordAlertAt,
lastDiscordAlertMeta,
cameras,
history: history.slice(),
updatedAt: now,
};
}
function emitStateToAdmins() {
const payload = buildState();
io.sockets.sockets.forEach((socket) => {
if (!isAdmin(socket)) return;
socket.emit('vision:human:state', payload);
});
} }
function evaluateEpisode(now = Date.now()) { function evaluateEpisode(now = Date.now()) {
if (!isDetectionActiveMode()) { if (!runtime.enabled) {
markCleared(now); markCleared(now, 'disabled');
return; return;
} }
if (episodeStartAt != null && lastAnyPositiveAt != null && now - lastAnyPositiveAt > CLEAR_WINDOW_MS) { if (!isDetectionActiveMode()) {
markCleared(now); markCleared(now, 'mode_gate');
return;
}
if (episodeStartAt != null && lastAnyPositiveAt != null && now - lastAnyPositiveAt > runtime.clearWindowMs) {
markCleared(now, 'clear_window');
return; return;
} }
if (episodeStartAt == null) return; if (episodeStartAt == null) return;
const elapsed = Math.max(0, now - episodeStartAt); const elapsed = Math.max(0, now - episodeStartAt);
if (!ttsSentThisEpisode && elapsed >= TTS_DELAY_MS) { if (!ttsSentThisEpisode && elapsed >= runtime.ttsDelayMs) {
sendTtsToNonPrivateRovers(); sendTtsToNonPrivateRovers();
ttsSentThisEpisode = true; ttsSentThisEpisode = true;
} }
if (discordSentThisEpisode) return; if (discordSentThisEpisode) return;
if (elapsed < DISCORD_DELAY_MS) return; if (elapsed < runtime.discordDelayMs) return;
if (!hasClearedSinceLastDiscord) return; if (!hasClearedSinceLastDiscord) return;
if (now < cooldownUntil) return; if (now < cooldownUntil) return;
sendDiscordDetectionAlert(now); sendDiscordDetectionAlert(now);
discordSentThisEpisode = true; discordSentThisEpisode = true;
hasClearedSinceLastDiscord = false; hasClearedSinceLastDiscord = false;
cooldownUntil = now + COOLDOWN_MS; cooldownUntil = now + runtime.cooldownMs;
} }
function handleWorkerMessage(line) { function handleWorkerMessage(line) {
@@ -175,7 +266,9 @@ function handleWorkerMessage(line) {
} }
if (msg?.type === 'ready') { if (msg?.type === 'ready') {
workerReady = true; workerReady = true;
pushHistory('worker.ready');
logger.info('Vision worker ready'); logger.info('Vision worker ready');
emitStateToAdmins();
return; return;
} }
const cameraId = String(msg?.cameraId || ''); const cameraId = String(msg?.cameraId || '');
@@ -187,6 +280,7 @@ function handleWorkerMessage(line) {
state.error = msg.error || 'worker error'; state.error = msg.error || 'worker error';
logger.warn('Vision inference failed', { cameraId, error: state.error }); logger.warn('Vision inference failed', { cameraId, error: state.error });
evaluateEpisode(Date.now()); evaluateEpisode(Date.now());
emitStateToAdmins();
return; return;
} }
state.error = null; state.error = null;
@@ -199,6 +293,7 @@ function handleWorkerMessage(line) {
episodeStartAt = now; episodeStartAt = now;
ttsSentThisEpisode = false; ttsSentThisEpisode = false;
discordSentThisEpisode = false; discordSentThisEpisode = false;
pushHistory('episode.started', { cameraId });
logger.info('Human detection episode started', { cameraId, now }); logger.info('Human detection episode started', { cameraId, now });
} }
let buffer = null; let buffer = null;
@@ -219,6 +314,7 @@ function handleWorkerMessage(line) {
} }
} }
evaluateEpisode(Number(msg.ts) || Date.now()); evaluateEpisode(Number(msg.ts) || Date.now());
emitStateToAdmins();
} }
function startWorker() { function startWorker() {
@@ -229,9 +325,8 @@ function startWorker() {
}); });
worker = proc; worker = proc;
workerReady = false; workerReady = false;
readline workerPython = pythonBin;
.createInterface({ input: proc.stdout }) readline.createInterface({ input: proc.stdout }).on('line', handleWorkerMessage);
.on('line', handleWorkerMessage);
proc.stderr.on('data', (chunk) => { proc.stderr.on('data', (chunk) => {
const text = String(chunk || '').trim(); const text = String(chunk || '').trim();
if (!text) return; if (!text) return;
@@ -241,8 +336,12 @@ function startWorker() {
logger.warn('Vision worker exited', { code, signal }); logger.warn('Vision worker exited', { code, signal });
worker = null; worker = null;
workerReady = false; workerReady = false;
workerRestartCount += 1;
pushHistory('worker.exit', { code, signal });
emitStateToAdmins();
}); });
logger.info('Vision worker started', { pythonBin, workerScript }); logger.info('Vision worker started', { pythonBin, workerScript });
pushHistory('worker.started', { pythonBin });
return true; return true;
} catch (err) { } catch (err) {
logger.warn('Failed to spawn vision worker candidate', { pythonBin, error: err.message }); logger.warn('Failed to spawn vision worker candidate', { pythonBin, error: err.message });
@@ -252,18 +351,19 @@ function startWorker() {
} }
function submitFrame(cameraId, buffer, ts = Date.now()) { function submitFrame(cameraId, buffer, ts = Date.now()) {
if (!runtime.enabled) return;
if (!worker || !workerReady) return; if (!worker || !workerReady) return;
const state = ensureCameraState(cameraId); const state = ensureCameraState(cameraId);
const now = Date.now(); const now = Date.now();
if (state.inflight) return; if (state.inflight) return;
if (now - state.lastInferAt < inferenceIntervalMs) return; if (now - state.lastInferAt < getInferenceIntervalMs()) return;
state.inflight = true; state.inflight = true;
state.lastInferAt = now; state.lastInferAt = now;
const payload = { const payload = {
reqId: `r${reqSeq++}`, reqId: `r${reqSeq++}`,
cameraId: String(cameraId), cameraId: String(cameraId),
ts: Number(ts) || now, ts: Number(ts) || now,
confidenceThreshold: CONFIDENCE_THRESHOLD, confidenceThreshold: runtime.confidenceThreshold,
imageBase64: buffer.toString('base64'), imageBase64: buffer.toString('base64'),
}; };
try { try {
@@ -275,12 +375,44 @@ function submitFrame(cameraId, buffer, ts = Date.now()) {
} }
} }
function clampNumber(value, min, max, fallback) {
const num = Number(value);
if (!Number.isFinite(num)) return fallback;
return Math.max(min, Math.min(max, num));
}
function updateRuntimeConfig(patch = {}) {
const prevEnabled = runtime.enabled;
runtime.enabled = typeof patch.enabled === 'boolean' ? patch.enabled : runtime.enabled;
runtime.confidenceThreshold = clampNumber(patch.confidenceThreshold, 0, 1, runtime.confidenceThreshold);
runtime.ttsDelayMs = clampNumber(patch.ttsDelayMs, 1000, 60 * 60 * 1000, runtime.ttsDelayMs);
runtime.discordDelayMs = clampNumber(
patch.discordDelayMs,
runtime.ttsDelayMs,
2 * 60 * 60 * 1000,
runtime.discordDelayMs,
);
runtime.clearWindowMs = clampNumber(patch.clearWindowMs, 250, 60 * 1000, runtime.clearWindowMs);
runtime.cooldownMs = clampNumber(patch.cooldownMs, 0, 12 * 60 * 60 * 1000, runtime.cooldownMs);
runtime.maxInferenceFpsPerCamera = clampNumber(
patch.maxInferenceFpsPerCamera,
0.25,
30,
runtime.maxInferenceFpsPerCamera,
);
if (prevEnabled && !runtime.enabled) {
markCleared(Date.now(), 'disabled');
}
pushHistory('config.updated', { patch });
}
if (!startWorker()) { if (!startWorker()) {
logger.error('Room human detection disabled; failed to start Python worker'); logger.error('Room human detection worker failed to start; detection unavailable until restart');
return; pushHistory('worker.unavailable');
} }
roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => { roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => {
if (!runtime.enabled) return;
if (!isDetectionActiveMode()) return; if (!isDetectionActiveMode()) return;
if (!id || !buffer) return; if (!id || !buffer) return;
submitFrame(String(id), buffer, ts); submitFrame(String(id), buffer, ts);
@@ -288,15 +420,88 @@ roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => {
setInterval(() => { setInterval(() => {
evaluateEpisode(Date.now()); evaluateEpisode(Date.now());
emitStateToAdmins();
}, 1000); }, 1000);
logger.info('Room human detection service started', { io.on('connection', (socket) => {
confidenceThreshold: CONFIDENCE_THRESHOLD, socket.on('vision:human:getState', (_, cb = () => {}) => {
ttsDelayMs: TTS_DELAY_MS, try {
discordDelayMs: DISCORD_DELAY_MS, if (!isAdmin(socket)) {
clearWindowMs: CLEAR_WINDOW_MS, cb({ error: 'Not authorized' });
cooldownMs: COOLDOWN_MS, return;
maxInferenceFpsPerCamera: MAX_INFERENCE_FPS, }
cb({ ok: true, state: buildState() });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('vision:human:updateConfig', ({ config: patch } = {}, cb = () => {}) => {
try {
if (!isAdmin(socket)) {
cb({ error: 'Not authorized' });
return;
}
updateRuntimeConfig(patch || {});
emitStateToAdmins();
cb({ ok: true, state: buildState() });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('vision:human:testTts', (_, cb = () => {}) => {
try {
if (!isAdmin(socket)) {
cb({ error: 'Not authorized' });
return;
}
sendTtsToNonPrivateRovers();
emitStateToAdmins();
cb({ ok: true, state: buildState() });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('vision:human:testDiscord', (_, cb = () => {}) => {
try {
if (!isAdmin(socket)) {
cb({ error: 'Not authorized' });
return;
}
sendDiscordDetectionAlert(Date.now(), { message: 'Human detection test alert.' });
emitStateToAdmins();
cb({ ok: true, state: buildState() });
} catch (err) {
cb({ error: err.message });
}
});
socket.on('vision:human:clear', (_, cb = () => {}) => {
try {
if (!isAdmin(socket)) {
cb({ error: 'Not authorized' });
return;
}
markCleared(Date.now(), 'manual_clear');
emitStateToAdmins();
cb({ ok: true, state: buildState() });
} catch (err) {
cb({ error: err.message });
}
});
if (isAdmin(socket)) {
socket.emit('vision:human:state', buildState());
}
}); });
module.exports = {}; logger.info('Room human detection service started', {
...runtime,
workerScript,
});
module.exports = {
getRoomHumanDetectionState: () => buildState(),
};
server/src/services/sessionService.js
+3 -1
View File
@@ -1,6 +1,6 @@
const io = require('../globals/io'); const io = require('../globals/io');
const logger = require('../globals/logger').child('sessionService'); const logger = require('../globals/logger').child('sessionService');
const { getRole, roleEvents } = require('./roleService'); const { getRole, isAdmin, roleEvents } = require('./roleService');
const { getMode, modeEvents } = require('./modeManager'); const { getMode, modeEvents } = require('./modeManager');
const roverManager = require('./roverManager'); const roverManager = require('./roverManager');
const { managerEvents } = roverManager; const { managerEvents } = roverManager;
@@ -28,6 +28,7 @@ const { subscribe } = require('./eventBus');
const { getSocketIp, isLocalNetwork } = require('../helpers/ipResolver'); const { getSocketIp, isLocalNetwork } = require('../helpers/ipResolver');
const { getAudioForwardState, audioForwardEvents } = require('./audioForwardService'); const { getAudioForwardState, audioForwardEvents } = require('./audioForwardService');
const { getAudioLevels, audioLevelsEvents } = require('./audioLevelsService'); const { getAudioLevels, audioLevelsEvents } = require('./audioLevelsService');
const { getRoomHumanDetectionState } = require('./roomHumanDetectionService');
const config = loadConfig(); const config = loadConfig();
const discordInvite = config.discord?.invite || null; const discordInvite = config.discord?.invite || null;
@@ -136,6 +137,7 @@ function buildSession(socket) {
isVerified: Boolean(socket?.data?.isVerified), isVerified: Boolean(socket?.data?.isVerified),
audioForward: getAudioForwardState(), audioForward: getAudioForwardState(),
audioLevels: getAudioLevels(), audioLevels: getAudioLevels(),
visionHumanDetection: isAdmin(socket) ? getRoomHumanDetectionState() : null,
}; };
} }
webui/src/components/AdminPanel.jsx
+266
View File
@@ -24,6 +24,12 @@ export default function AdminPanel() {
setAudioLevels, setAudioLevels,
setPrivateSafety, setPrivateSafety,
llmControl, llmControl,
visionHumanState,
getVisionHumanState,
updateVisionHumanConfig,
testVisionHumanTts,
testVisionHumanDiscord,
clearVisionHumanState,
adminLogs, adminLogs,
llmCommentaryState, llmCommentaryState,
} = useSession(); } = useSession();
@@ -46,6 +52,7 @@ export default function AdminPanel() {
forwardGain: Number.isFinite(currentAudioLevels.forwardGain) ? currentAudioLevels.forwardGain : 1, forwardGain: Number.isFinite(currentAudioLevels.forwardGain) ? currentAudioLevels.forwardGain : 1,
}); });
const [privateSafetyDrafts, setPrivateSafetyDrafts] = useState({}); const [privateSafetyDrafts, setPrivateSafetyDrafts] = useState({});
const [visionDraft, setVisionDraft] = useState(null);
const isAdmin = const isAdmin =
session?.role === 'admin' || session?.role === 'admin' ||
@@ -183,6 +190,24 @@ export default function AdminPanel() {
}); });
}, [currentAudioLevels.forwardGain, currentAudioLevels.hornGain, currentAudioLevels.ttsGain]); }, [currentAudioLevels.forwardGain, currentAudioLevels.hornGain, currentAudioLevels.ttsGain]);
useEffect(() => {
const source = visionHumanState?.config;
if (!source) return;
setVisionDraft({
enabled: Boolean(visionHumanState?.enabled),
confidenceThreshold: Number.isFinite(Number(source.confidenceThreshold))
? Number(source.confidenceThreshold)
: 0.55,
ttsDelayMs: Number.isFinite(Number(source.ttsDelayMs)) ? Number(source.ttsDelayMs) : 30000,
discordDelayMs: Number.isFinite(Number(source.discordDelayMs)) ? Number(source.discordDelayMs) : 60000,
clearWindowMs: Number.isFinite(Number(source.clearWindowMs)) ? Number(source.clearWindowMs) : 3000,
cooldownMs: Number.isFinite(Number(source.cooldownMs)) ? Number(source.cooldownMs) : 900000,
maxInferenceFpsPerCamera: Number.isFinite(Number(source.maxInferenceFpsPerCamera))
? Number(source.maxInferenceFpsPerCamera)
: 3,
});
}, [visionHumanState]);
useEffect(() => { useEffect(() => {
const next = {}; const next = {};
(roster || []).forEach((rover) => { (roster || []).forEach((rover) => {
@@ -243,6 +268,54 @@ export default function AdminPanel() {
} }
}; };
const handleRefreshVision = async () => {
try {
await getVisionHumanState();
} catch (err) {
alert(err.message);
}
};
const handleVisionDraftChange = (key) => (event) => {
if (!visionDraft) return;
const isCheckbox = event.target.type === 'checkbox';
const value = isCheckbox ? Boolean(event.target.checked) : Number(event.target.value);
setVisionDraft((current) => ({ ...(current || {}), [key]: isCheckbox ? value : value }));
};
const handleSaveVisionConfig = async () => {
if (!visionDraft) return;
try {
await updateVisionHumanConfig(visionDraft);
} catch (err) {
alert(err.message);
}
};
const handleVisionTestTts = async () => {
try {
await testVisionHumanTts();
} catch (err) {
alert(err.message);
}
};
const handleVisionTestDiscord = async () => {
try {
await testVisionHumanDiscord();
} catch (err) {
alert(err.message);
}
};
const handleVisionClear = async () => {
try {
await clearVisionHumanState();
} catch (err) {
alert(err.message);
}
};
if (!isAdmin) return null; if (!isAdmin) return null;
return ( return (
@@ -462,6 +535,16 @@ export default function AdminPanel() {
)} )}
/> />
<ReplaySnapshotHealth health={health} roster={roster} /> <ReplaySnapshotHealth health={health} roster={roster} />
<VisionHumanPanel
state={visionHumanState}
draft={visionDraft}
onDraftChange={handleVisionDraftChange}
onRefresh={handleRefreshVision}
onSaveConfig={handleSaveVisionConfig}
onTestTts={handleVisionTestTts}
onTestDiscord={handleVisionTestDiscord}
onClear={handleVisionClear}
/>
<LlmCommentaryPanel <LlmCommentaryPanel
state={llmCommentaryState} state={llmCommentaryState}
onClearHistory={handleClearLlmHistory} onClearHistory={handleClearLlmHistory}
@@ -472,6 +555,189 @@ export default function AdminPanel() {
); );
} }
function VisionHumanPanel({
state,
draft,
onDraftChange,
onRefresh,
onSaveConfig,
onTestTts,
onTestDiscord,
onClear,
}) {
if (!state) {
return (
<div className="space-y-0.5">
<div className="panel-muted text-xs uppercase">Human Detection</div>
<div className="surface text-xs text-slate-300">No human detection state received yet.</div>
</div>
);
}
const modeGateText = state.modeActive ? 'active' : `inactive in ${state.mode}`;
const workerText = state.workerReady ? 'ready' : state.workerRunning ? 'starting' : 'offline';
const episode = state.episode || {};
const history = Array.isArray(state.history) ? state.history : [];
const cameras = Array.isArray(state.cameras) ? state.cameras : [];
const statusPills = [
{ label: 'enabled', value: state.enabled ? 'yes' : 'no' },
{ label: 'mode gate', value: modeGateText },
{ label: 'worker', value: workerText },
{ label: 'python', value: state.workerPython || '--' },
{ label: 'worker restarts', value: state.workerRestartCount ?? 0 },
{ label: 'present', value: episode.humanPresent ? 'yes' : 'no' },
{ label: 'tts sent', value: episode.ttsSentThisEpisode ? 'yes' : 'no' },
{ label: 'discord sent', value: episode.discordSentThisEpisode ? 'yes' : 'no' },
{ label: 'tts in', value: `${Math.ceil((episode.timeToTtsMs || 0) / 1000)}s` },
{ label: 'discord in', value: `${Math.ceil((episode.timeToDiscordMs || 0) / 1000)}s` },
{ label: 'cooldown', value: `${Math.ceil((episode.cooldownRemainingMs || 0) / 1000)}s` },
];
return (
<div className="space-y-0.5">
<div className="panel-muted text-xs uppercase">Human Detection</div>
<div className="flex flex-wrap gap-0.5 text-xs">
<button type="button" onClick={onRefresh} className="button-dark">
Refresh
</button>
<button type="button" onClick={onSaveConfig} className="button-dark">
Save Detection Config
</button>
<button type="button" onClick={onTestTts} className="button-dark">
Test TTS
</button>
<button type="button" onClick={onTestDiscord} className="button-dark">
Test Discord
</button>
<button type="button" onClick={onClear} className="button-danger">
Clear Episode
</button>
</div>
<div className="surface flex flex-wrap gap-0.5 text-xs">
{statusPills.map((pill) => (
<span
key={pill.label}
className="rounded border border-slate-600/60 bg-slate-800/70 px-0.5 py-0.25 text-[0.72rem] leading-tight text-slate-200"
>
{pill.label}: {pill.value}
</span>
))}
</div>
{draft ? (
<div className="grid gap-0.5 md:grid-cols-2">
<label className="surface flex items-center justify-between gap-0.5 text-xs">
<span>Enabled</span>
<input type="checkbox" checked={Boolean(draft.enabled)} onChange={onDraftChange('enabled')} />
</label>
<label className="surface grid gap-0.25 text-xs">
<span>Confidence threshold</span>
<input
type="number"
min="0"
max="1"
step="0.01"
value={draft.confidenceThreshold}
onChange={onDraftChange('confidenceThreshold')}
className="field-input text-xs"
/>
</label>
<label className="surface grid gap-0.25 text-xs">
<span>TTS delay (ms)</span>
<input
type="number"
min="1000"
step="250"
value={draft.ttsDelayMs}
onChange={onDraftChange('ttsDelayMs')}
className="field-input text-xs"
/>
</label>
<label className="surface grid gap-0.25 text-xs">
<span>Discord delay (ms)</span>
<input
type="number"
min="1000"
step="250"
value={draft.discordDelayMs}
onChange={onDraftChange('discordDelayMs')}
className="field-input text-xs"
/>
</label>
<label className="surface grid gap-0.25 text-xs">
<span>Clear window (ms)</span>
<input
type="number"
min="250"
step="250"
value={draft.clearWindowMs}
onChange={onDraftChange('clearWindowMs')}
className="field-input text-xs"
/>
</label>
<label className="surface grid gap-0.25 text-xs">
<span>Cooldown (ms)</span>
<input
type="number"
min="0"
step="1000"
value={draft.cooldownMs}
onChange={onDraftChange('cooldownMs')}
className="field-input text-xs"
/>
</label>
<label className="surface grid gap-0.25 text-xs">
<span>Max inference FPS/camera</span>
<input
type="number"
min="0.25"
max="30"
step="0.25"
value={draft.maxInferenceFpsPerCamera}
onChange={onDraftChange('maxInferenceFpsPerCamera')}
className="field-input text-xs"
/>
</label>
</div>
) : null}
<div className="space-y-0.5">
<div className="panel-muted text-xs uppercase">Per Camera</div>
<div className="surface max-h-40 overflow-y-auto text-xs text-slate-200">
{cameras.length ? (
cameras.map((cam) => (
<div key={cam.cameraId} className="flex items-center justify-between gap-0.5">
<span>{cam.cameraId}</span>
<span>{cam.lastConfidence?.toFixed?.(2) ?? '0.00'}</span>
<span>{cam.lastPositiveAt ? new Date(cam.lastPositiveAt).toLocaleTimeString() : '--'}</span>
<span className={cam.error ? 'text-amber-300' : 'text-emerald-300'}>
{cam.error ? 'error' : cam.inflight ? 'inference' : 'ok'}
</span>
</div>
))
) : (
<div className="text-slate-400">No camera state yet.</div>
)}
</div>
</div>
<div className="space-y-0.5">
<div className="panel-muted text-xs uppercase">Recent Events</div>
<div className="surface max-h-40 overflow-y-auto text-xs text-slate-200">
{history.length ? (
history
.slice()
.reverse()
.map((entry, idx) => (
<div key={`${entry.ts}-${entry.type}-${idx}`} className="flex items-start justify-between gap-0.5">
<span>{entry.type}</span>
<span className="text-slate-400">{entry.ts ? new Date(entry.ts).toLocaleTimeString() : '--'}</span>
</div>
))
) : (
<div className="text-slate-400">No events yet.</div>
)}
</div>
</div>
</div>
);
}
function LlmCommentaryPanel({ state, onClearHistory, clearingHistory }) { function LlmCommentaryPanel({ state, onClearHistory, clearingHistory }) {
const [selectedRunId, setSelectedRunId] = useState(null); const [selectedRunId, setSelectedRunId] = useState(null);
if (!state) { if (!state) {
webui/src/context/SessionContext.jsx
+33 -1
View File
@@ -10,6 +10,7 @@ const SessionContext = createContext({
adminLogs: [], adminLogs: [],
llmCommentaryState: null, llmCommentaryState: null,
llmCommentaryStatus: null, llmCommentaryStatus: null,
visionHumanState: null,
identifySession: async () => {}, identifySession: async () => {},
login: async () => {}, login: async () => {},
setRole: async () => {}, setRole: async () => {},
@@ -35,6 +36,11 @@ const SessionContext = createContext({
setAudioLevels: async () => {}, setAudioLevels: async () => {},
setPrivateSafety: async () => {}, setPrivateSafety: async () => {},
llmControl: async () => {}, llmControl: async () => {},
getVisionHumanState: async () => {},
updateVisionHumanConfig: async () => {},
testVisionHumanTts: async () => {},
testVisionHumanDiscord: async () => {},
clearVisionHumanState: async () => {},
}); });
function useAckEmitter(socket) { function useAckEmitter(socket) {
@@ -61,6 +67,7 @@ export function SessionProvider({ children }) {
const [adminLogs, setAdminLogs] = useState([]); const [adminLogs, setAdminLogs] = useState([]);
const [llmCommentaryState, setLlmCommentaryState] = useState(null); const [llmCommentaryState, setLlmCommentaryState] = useState(null);
const [llmCommentaryStatus, setLlmCommentaryStatus] = useState(null); const [llmCommentaryStatus, setLlmCommentaryStatus] = useState(null);
const [visionHumanState, setVisionHumanState] = useState(null);
const [alerts, setAlerts] = useState([]); const [alerts, setAlerts] = useState([]);
const [connected, setConnected] = useState(socket.connected); const [connected, setConnected] = useState(socket.connected);
@@ -78,6 +85,9 @@ export function SessionProvider({ children }) {
useEffect(() => { useEffect(() => {
function handleSession(payload) { function handleSession(payload) {
setSession(payload); setSession(payload);
if (payload?.visionHumanDetection) {
setVisionHumanState(payload.visionHumanDetection);
}
} }
function handleLogInit(entries = []) { function handleLogInit(entries = []) {
setLogs(entries); setLogs(entries);
@@ -104,6 +114,10 @@ export function SessionProvider({ children }) {
socket.on('adminlog:init', handleAdminLogInit); socket.on('adminlog:init', handleAdminLogInit);
socket.on('adminlog:entry', handleAdminLogEntry); socket.on('adminlog:entry', handleAdminLogEntry);
socket.on('llm:state', handleLlmState); socket.on('llm:state', handleLlmState);
socket.on('vision:human:state', (payload = null) => {
const state = payload && typeof payload === 'object' ? payload : null;
setVisionHumanState(state);
});
socket.on('alert:new', (payload = {}) => { socket.on('alert:new', (payload = {}) => {
setAlerts((prev) => [ setAlerts((prev) => [
...prev.slice(-49), ...prev.slice(-49),
@@ -120,6 +134,7 @@ export function SessionProvider({ children }) {
socket.off('adminlog:init', handleAdminLogInit); socket.off('adminlog:init', handleAdminLogInit);
socket.off('adminlog:entry', handleAdminLogEntry); socket.off('adminlog:entry', handleAdminLogEntry);
socket.off('llm:state', handleLlmState); socket.off('llm:state', handleLlmState);
socket.off('vision:human:state');
socket.off('alert:new'); socket.off('alert:new');
}; };
}, [socket]); }, [socket]);
@@ -162,6 +177,12 @@ export function SessionProvider({ children }) {
emitWithAck('session:privateSafety:set', { roverId, safety }), emitWithAck('session:privateSafety:set', { roverId, safety }),
llmControl: (action, controls = {}) => llmControl: (action, controls = {}) =>
emitWithAck('llm:control', { controls: { action, ...controls } }), emitWithAck('llm:control', { controls: { action, ...controls } }),
getVisionHumanState: () => emitWithAck('vision:human:getState'),
updateVisionHumanConfig: (nextConfig = {}) =>
emitWithAck('vision:human:updateConfig', { config: nextConfig }),
testVisionHumanTts: () => emitWithAck('vision:human:testTts'),
testVisionHumanDiscord: () => emitWithAck('vision:human:testDiscord'),
clearVisionHumanState: () => emitWithAck('vision:human:clear'),
pushAlert: (alert) => pushAlert: (alert) =>
setAlerts((prev) => [ setAlerts((prev) => [
...prev.slice(-49), ...prev.slice(-49),
@@ -179,10 +200,21 @@ export function SessionProvider({ children }) {
adminLogs, adminLogs,
llmCommentaryState, llmCommentaryState,
llmCommentaryStatus, llmCommentaryStatus,
visionHumanState,
alerts, alerts,
...actions, ...actions,
}), }),
[actions, adminLogs, alerts, connected, llmCommentaryState, llmCommentaryStatus, logs, session], [
actions,
adminLogs,
alerts,
connected,
llmCommentaryState,
llmCommentaryStatus,
logs,
session,
visionHumanState,
],
); );
return <SessionContext.Provider value={value}>{children}</SessionContext.Provider>; return <SessionContext.Provider value={value}>{children}</SessionContext.Provider>;