mirror of
https://github.com/legop3/MultiRoombaRover.git
synced 2026-09-16 09:31:20 -04:00
switch human alert to button
This commit is contained in:
+11
-15
@@ -40,6 +40,17 @@ homeAssistant:
|
||||
- id: "switch.dock_power"
|
||||
name: "Dock Power"
|
||||
# type is optional; if omitted it is inferred from the entity id (light/switch)
|
||||
buttons:
|
||||
# All Home Assistant button automations live here.
|
||||
- entityId: "sensor.human_alert_button_action"
|
||||
stateEquals: "single"
|
||||
cooldownMs: 15000
|
||||
action: "humanAlert"
|
||||
# Example future button:
|
||||
# - entityId: "sensor.replay_button_action"
|
||||
# stateEquals: "single"
|
||||
# cooldownMs: 10000
|
||||
# action: "replayClip"
|
||||
roomCameras:
|
||||
- id: "lobby"
|
||||
name: "Lobby Camera"
|
||||
@@ -52,21 +63,6 @@ roomCameras:
|
||||
url: "http://192.168.0.51/snapshot.jpg"
|
||||
streamUrl: "http://192.168.0.51/stream.mjpg"
|
||||
|
||||
vision:
|
||||
humanDetection:
|
||||
enabled: false
|
||||
confidenceThreshold: 0.55
|
||||
# Continuous presence windows:
|
||||
# - 30s => rover TTS warning
|
||||
# - 60s => Discord human alert
|
||||
ttsDelayMs: 30000
|
||||
discordDelayMs: 60000
|
||||
# Tolerates brief detector misses before declaring "cleared".
|
||||
clearWindowMs: 3000
|
||||
# After a Discord alert, require clear + cooldown before next alert.
|
||||
cooldownMs: 900000
|
||||
maxInferenceFpsPerCamera: 3
|
||||
|
||||
discord:
|
||||
token: "DISCORD_BOT_TOKEN"
|
||||
guildId: "123456789012345678" # optional; bot works in any guild it's invited to
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ require('./src/services/videoAuthService');
|
||||
require('./src/services/videoSocketService');
|
||||
require('./src/services/roomCameraSocketService');
|
||||
require('./src/services/roverSnapshotSocketService');
|
||||
require('./src/services/roomHumanDetectionService');
|
||||
require('./src/services/humanAlertButtonService');
|
||||
require('./src/services/embedHttpService');
|
||||
require('./src/services/logStreamService');
|
||||
require('./src/services/adminLogService');
|
||||
|
||||
@@ -10,9 +10,6 @@ MEDIAMTX_SERVICE="/etc/systemd/system/mediamtx.service"
|
||||
MULTIROVER_SERVICE="/etc/systemd/system/multirover.service"
|
||||
SNAPSHOT_DIR="/var/lib/rover-snapshots"
|
||||
REPLAY_SEGMENT_DIR="/var/lib/replay-segments"
|
||||
VISION_HOME="/opt/multiroomba-vision"
|
||||
VISION_VENV="$VISION_HOME/.venv"
|
||||
VISION_PYTHON="$VISION_VENV/bin/python3"
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This installer must be run with sudo/root." >&2
|
||||
@@ -30,11 +27,11 @@ SERVER_DIR="$SCRIPT_DIR"
|
||||
CONFIG_PATH="$SERVER_DIR/config.yaml"
|
||||
MEDIAMTX_TEMPLATE="$SERVER_DIR/mediamtx/mediamtx.yml"
|
||||
|
||||
echo "[1/7] Installing dependencies..."
|
||||
dnf install -y nodejs npm curl tar python3 python3-pip >/dev/null
|
||||
echo "[1/6] Installing dependencies..."
|
||||
dnf install -y nodejs npm curl tar >/dev/null
|
||||
NODE_BIN="$(command -v node)"
|
||||
|
||||
echo "[2/7] Installing Node production deps..."
|
||||
echo "[2/6] Installing Node production deps..."
|
||||
runuser -u "$TARGET_USER" -- bash -c "cd '$SERVER_DIR' && npm install --production"
|
||||
|
||||
if [[ ! -f "$CONFIG_PATH" ]]; then
|
||||
@@ -43,21 +40,6 @@ if [[ ! -f "$CONFIG_PATH" ]]; then
|
||||
echo "Copied config.example.yaml to config.yaml; edit it before exposing the service."
|
||||
fi
|
||||
|
||||
echo "[3/7] Installing OpenCV runtime..."
|
||||
mkdir -p "$VISION_HOME"
|
||||
python3 -m venv "$VISION_VENV"
|
||||
"$VISION_PYTHON" -m pip install --upgrade pip >/dev/null
|
||||
"$VISION_PYTHON" -m pip install \
|
||||
--only-binary=:all: \
|
||||
"numpy>=2.1,<3" \
|
||||
"opencv-python-headless>=4.10,<5" >/dev/null
|
||||
"$VISION_PYTHON" - <<'PY'
|
||||
import cv2
|
||||
import numpy
|
||||
print("Vision runtime OK", cv2.__version__, numpy.__version__)
|
||||
PY
|
||||
chown -R "$TARGET_USER":"$TARGET_USER" "$VISION_HOME"
|
||||
|
||||
tmpdir=$(mktemp -d)
|
||||
trap 'rm -rf "$tmpdir"' EXIT
|
||||
|
||||
@@ -78,7 +60,7 @@ case "$arch" in
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "[4/7] Installing mediaMTX ${MEDIAMTX_VERSION}..."
|
||||
echo "[3/6] Installing mediaMTX ${MEDIAMTX_VERSION}..."
|
||||
curl -L "$MEDIAMTX_BASE_URL/$mediamtx_pkg" -o "$tmpdir/mediamtx.tgz"
|
||||
tar -xzf "$tmpdir/mediamtx.tgz" -C "$tmpdir" mediamtx
|
||||
install -m 0755 "$tmpdir/mediamtx" "$MEDIAMTX_BIN"
|
||||
@@ -93,7 +75,7 @@ rm -f "$MEDIAMTX_CONFIG"
|
||||
install -m 0644 "$MEDIAMTX_TEMPLATE" "$MEDIAMTX_CONFIG"
|
||||
chown -R "$TARGET_USER":"$TARGET_USER" "$MEDIAMTX_CONF_DIR"
|
||||
|
||||
echo "[5/7] Writing systemd units..."
|
||||
echo "[4/6] Writing systemd units..."
|
||||
mkdir -p "$SNAPSHOT_DIR"
|
||||
chown "$TARGET_USER":"$TARGET_USER" "$SNAPSHOT_DIR"
|
||||
mkdir -p "$REPLAY_SEGMENT_DIR"
|
||||
@@ -131,7 +113,6 @@ Environment=NODE_ENV=production
|
||||
Environment=SERVER_CONFIG=$CONFIG_PATH
|
||||
Environment=ROVER_SNAPSHOT_DIR=$SNAPSHOT_DIR
|
||||
Environment=REPLAY_SEGMENT_DIR=$REPLAY_SEGMENT_DIR
|
||||
Environment=VISION_PYTHON=$VISION_PYTHON
|
||||
ExecStart=$NODE_BIN $SERVER_DIR/index.js
|
||||
Restart=on-failure
|
||||
RestartSec=2
|
||||
@@ -142,14 +123,14 @@ EOF
|
||||
|
||||
chmod 644 "$MEDIAMTX_SERVICE" "$MULTIROVER_SERVICE"
|
||||
|
||||
echo "[6/7] Enabling services..."
|
||||
echo "[5/6] Enabling services..."
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now mediamtx.service
|
||||
systemctl enable --now multirover.service
|
||||
systemctl restart mediamtx.service
|
||||
systemctl restart multirover.service
|
||||
|
||||
echo "[7/7] Done."
|
||||
echo "[6/6] Done."
|
||||
echo
|
||||
echo "Services installed:"
|
||||
echo " mediamtx.service (WebRTC fan-out)"
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -11,7 +11,7 @@
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
|
||||
<title>Multi Roomba Rover</title>
|
||||
<script type="module" crossorigin src="/assets/index-oXFJbPgP.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-dMtKgicC.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-BPze86Ff.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
MAX_EDGE = 640
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def emit(payload):
|
||||
sys.stdout.write(json.dumps(payload, separators=(",", ":")) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def decode_image(image_b64):
|
||||
raw = base64.b64decode(image_b64)
|
||||
arr = np.frombuffer(raw, dtype=np.uint8)
|
||||
frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
|
||||
if frame is None:
|
||||
raise ValueError("failed to decode image")
|
||||
return frame
|
||||
|
||||
|
||||
def resize_for_inference(frame):
|
||||
h, w = frame.shape[:2]
|
||||
max_side = max(h, w)
|
||||
if max_side <= MAX_EDGE:
|
||||
return frame
|
||||
scale = float(MAX_EDGE) / float(max_side)
|
||||
nw = max(1, int(round(w * scale)))
|
||||
nh = max(1, int(round(h * scale)))
|
||||
return cv2.resize(frame, (nw, nh), interpolation=cv2.INTER_AREA)
|
||||
|
||||
|
||||
def encode_jpeg(frame):
|
||||
ok, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 78])
|
||||
if not ok:
|
||||
raise ValueError("failed to encode image")
|
||||
return base64.b64encode(encoded.tobytes()).decode("ascii")
|
||||
|
||||
|
||||
def annotate_frame(frame, camera_id, confidence):
|
||||
stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
msg = f"person {confidence:.2f} | {camera_id} | {stamp}"
|
||||
cv2.rectangle(frame, (8, 8), (min(frame.shape[1] - 8, 540), 42), (0, 0, 0), -1)
|
||||
cv2.putText(
|
||||
frame,
|
||||
msg,
|
||||
(14, 30),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.6,
|
||||
(0, 255, 0),
|
||||
2,
|
||||
cv2.LINE_AA,
|
||||
)
|
||||
return frame
|
||||
|
||||
|
||||
hog = cv2.HOGDescriptor()
|
||||
hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
|
||||
|
||||
emit({"type": "ready", "ts": now_iso()})
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
req = None
|
||||
try:
|
||||
req = json.loads(line)
|
||||
req_id = req.get("reqId")
|
||||
cam_id = str(req.get("cameraId", "unknown"))
|
||||
conf_threshold = float(req.get("confidenceThreshold", 0.55))
|
||||
frame = decode_image(req["imageBase64"])
|
||||
working = resize_for_inference(frame)
|
||||
rects, weights = hog.detectMultiScale(
|
||||
working,
|
||||
winStride=(8, 8),
|
||||
padding=(8, 8),
|
||||
scale=1.05,
|
||||
)
|
||||
best = 0.0
|
||||
detections = []
|
||||
if weights is None:
|
||||
weights = []
|
||||
for i, rect in enumerate(rects):
|
||||
weight = float(weights[i]) if i < len(weights) else 0.0
|
||||
best = max(best, weight)
|
||||
if weight < conf_threshold:
|
||||
continue
|
||||
x, y, w, h = rect
|
||||
detections.append({"x": int(x), "y": int(y), "w": int(w), "h": int(h), "confidence": weight})
|
||||
cv2.rectangle(working, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 0), 2)
|
||||
cv2.putText(
|
||||
working,
|
||||
f"{weight:.2f}",
|
||||
(int(x), max(12, int(y) - 6)),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.5,
|
||||
(0, 255, 0),
|
||||
1,
|
||||
cv2.LINE_AA,
|
||||
)
|
||||
detected = len(detections) > 0
|
||||
if detected:
|
||||
working = annotate_frame(working, cam_id, max(d["confidence"] for d in detections))
|
||||
emit(
|
||||
{
|
||||
"ok": True,
|
||||
"reqId": req_id,
|
||||
"cameraId": cam_id,
|
||||
"personDetected": detected,
|
||||
"bestConfidence": best,
|
||||
"detections": detections,
|
||||
"annotatedBase64": encode_jpeg(working),
|
||||
"ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000),
|
||||
}
|
||||
)
|
||||
except Exception as err:
|
||||
emit(
|
||||
{
|
||||
"ok": False,
|
||||
"reqId": req.get("reqId") if isinstance(req, dict) else None,
|
||||
"error": str(err),
|
||||
"ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000),
|
||||
}
|
||||
)
|
||||
@@ -1460,37 +1460,32 @@ function handleBusEvent(event) {
|
||||
logger.warn('Replay send failed', err.message);
|
||||
});
|
||||
break;
|
||||
case 'vision.humanDetected': {
|
||||
case 'humanAlert.buttonPressed': {
|
||||
const imageBase64 = payload?.imageBase64 ? String(payload.imageBase64) : '';
|
||||
let attachment = null;
|
||||
if (imageBase64) {
|
||||
try {
|
||||
const imageBuffer = Buffer.from(imageBase64, 'base64');
|
||||
if (imageBuffer.length > 0) {
|
||||
attachment = new AttachmentBuilder(imageBuffer, { name: 'human-detected.jpg' });
|
||||
attachment = new AttachmentBuilder(imageBuffer, { name: 'human-alert-mosaic.jpg' });
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to decode human detection image for Discord', err.message);
|
||||
logger.warn('Failed to decode human alert image for Discord', err.message);
|
||||
}
|
||||
}
|
||||
const cameraLabel = payload?.cameraId ? `Camera: \`${payload.cameraId}\`` : 'Camera: unknown';
|
||||
const confidenceLabel =
|
||||
Number.isFinite(Number(payload?.confidence))
|
||||
? `Confidence: \`${Number(payload.confidence).toFixed(2)}\``
|
||||
: 'Confidence: n/a';
|
||||
const detectedAt = Number(payload?.detectedAt);
|
||||
const detectedLabel = Number.isFinite(detectedAt)
|
||||
? `Detected: <t:${Math.floor(detectedAt / 1000)}:F>`
|
||||
const triggeredAt = Number(payload?.triggeredAt);
|
||||
const triggeredLabel = Number.isFinite(triggeredAt)
|
||||
? `Triggered: <t:${Math.floor(triggeredAt / 1000)}:F>`
|
||||
: null;
|
||||
const embed = buildEmbed({
|
||||
title: 'Human Detected',
|
||||
description: [cameraLabel, confidenceLabel, detectedLabel].filter(Boolean).join('\n'),
|
||||
title: 'Human Alert Button Pressed',
|
||||
description: [triggeredLabel].filter(Boolean).join('\n'),
|
||||
color: 0xe53935,
|
||||
});
|
||||
announce({
|
||||
channelId: channels.humanAlerts,
|
||||
pingRoleId: roles.humanAlertPing || null,
|
||||
content: payload?.message || 'Human detected in room.',
|
||||
content: payload?.message || 'Human alert button pressed.',
|
||||
embeds: [embed],
|
||||
files: attachment ? [attachment] : [],
|
||||
includeSiteUrl: false,
|
||||
|
||||
@@ -6,6 +6,7 @@ const logger = require('../globals/logger').child('homeAssistantService');
|
||||
const { loadConfig } = require('../helpers/configLoader');
|
||||
const { getMode } = require('./modeManager');
|
||||
const { isAdmin, isLockdownAdmin } = require('./roleService');
|
||||
const { publishEvent } = require('./eventBus');
|
||||
|
||||
// home-assistant-js-websocket expects a global WebSocket in Node.
|
||||
if (!global.WebSocket) {
|
||||
@@ -18,6 +19,9 @@ const haConfig = config.homeAssistant || {};
|
||||
const events = new EventEmitter();
|
||||
const entityConfig = new Map(); // entityId -> { id, name, type }
|
||||
const entityState = new Map(); // entityId -> normalized state
|
||||
const triggerConfig = []; // [{ runtimeKey, entityId, action, stateEquals, payload, cooldownMs, allowedModes }]
|
||||
const triggerRuntime = new Map(); // triggerId -> { lastFiredAt, lastState, lastChanged, lastUpdated }
|
||||
const HA_BUTTON_EVENT_TYPE = 'ha.button.action';
|
||||
|
||||
let connection = null;
|
||||
let unsubscribeEntities = null;
|
||||
@@ -78,6 +82,49 @@ function loadEntityConfig() {
|
||||
logger.info('Loaded Home Assistant entities', { count: entityConfig.size });
|
||||
}
|
||||
|
||||
function normalizeTriggerEntry(entry, index) {
|
||||
if (!entry || typeof entry !== 'object') return null;
|
||||
const entityId = String(entry.entityId || entry.entity_id || '').trim();
|
||||
const action = String(entry.action || '').trim();
|
||||
if (!entityId || !action) return null;
|
||||
const stateEqualsRaw = entry.stateEquals ?? entry.state_equals;
|
||||
const stateEquals =
|
||||
stateEqualsRaw === null || stateEqualsRaw === undefined ? null : String(stateEqualsRaw).trim();
|
||||
const runtimeKey = `${action}::${entityId}::${stateEquals || '*'}::${index}`;
|
||||
const cooldownMs = Number.isFinite(Number(entry.cooldownMs)) ? Math.max(0, Number(entry.cooldownMs)) : 0;
|
||||
const allowedModes = Array.isArray(entry.allowedModes)
|
||||
? entry.allowedModes.map((mode) => String(mode || '').trim().toLowerCase()).filter(Boolean)
|
||||
: null;
|
||||
return {
|
||||
runtimeKey,
|
||||
entityId,
|
||||
action,
|
||||
stateEquals,
|
||||
payload: entry.payload && typeof entry.payload === 'object' ? entry.payload : {},
|
||||
cooldownMs,
|
||||
allowedModes,
|
||||
};
|
||||
}
|
||||
|
||||
function loadTriggerConfig() {
|
||||
triggerConfig.length = 0;
|
||||
const list = Array.isArray(haConfig?.buttons) ? haConfig.buttons : [];
|
||||
list.forEach((entry, index) => {
|
||||
const normalized = normalizeTriggerEntry(entry, index);
|
||||
if (!normalized) return;
|
||||
triggerConfig.push(normalized);
|
||||
if (!triggerRuntime.has(normalized.runtimeKey)) {
|
||||
triggerRuntime.set(normalized.runtimeKey, {
|
||||
lastFiredAt: 0,
|
||||
lastState: null,
|
||||
lastChanged: null,
|
||||
lastUpdated: null,
|
||||
});
|
||||
}
|
||||
});
|
||||
logger.info('Loaded Home Assistant buttons', { count: triggerConfig.length });
|
||||
}
|
||||
|
||||
function buildState(meta, raw) {
|
||||
if (!meta) return null;
|
||||
const name = meta.name || raw?.attributes?.friendly_name || meta.id;
|
||||
@@ -153,6 +200,75 @@ function handleEntitySnapshot(snapshot = {}) {
|
||||
if (changed) {
|
||||
emitUpdate();
|
||||
}
|
||||
evaluateTriggers(snapshot);
|
||||
}
|
||||
|
||||
function triggerMatches(trigger, raw, runtimeState) {
|
||||
if (!raw) return false;
|
||||
const nextState = raw?.state ?? null;
|
||||
const nextChanged = raw?.last_changed ?? null;
|
||||
const nextUpdated = raw?.last_updated ?? null;
|
||||
const changed =
|
||||
runtimeState.lastState !== nextState ||
|
||||
runtimeState.lastChanged !== nextChanged ||
|
||||
runtimeState.lastUpdated !== nextUpdated;
|
||||
if (!changed) {
|
||||
return { matched: false, nextState, nextChanged, nextUpdated };
|
||||
}
|
||||
if (trigger.stateEquals != null && String(trigger.stateEquals) !== String(nextState)) {
|
||||
return { matched: false, nextState, nextChanged, nextUpdated };
|
||||
}
|
||||
return { matched: true, nextState, nextChanged, nextUpdated };
|
||||
}
|
||||
|
||||
function evaluateTriggers(snapshot = {}) {
|
||||
if (!triggerConfig.length) return;
|
||||
const now = Date.now();
|
||||
const mode = String(getMode() || '').toLowerCase();
|
||||
triggerConfig.forEach((trigger) => {
|
||||
const runtimeState = triggerRuntime.get(trigger.runtimeKey) || {
|
||||
lastFiredAt: 0,
|
||||
lastState: null,
|
||||
lastChanged: null,
|
||||
lastUpdated: null,
|
||||
};
|
||||
const raw = snapshot?.[trigger.entityId] || null;
|
||||
const evalResult = triggerMatches(trigger, raw, runtimeState);
|
||||
runtimeState.lastState = evalResult.nextState;
|
||||
runtimeState.lastChanged = evalResult.nextChanged;
|
||||
runtimeState.lastUpdated = evalResult.nextUpdated;
|
||||
triggerRuntime.set(trigger.runtimeKey, runtimeState);
|
||||
if (!evalResult.matched) return;
|
||||
if (trigger.allowedModes?.length && !trigger.allowedModes.includes(mode)) return;
|
||||
if (trigger.cooldownMs > 0 && now - runtimeState.lastFiredAt < trigger.cooldownMs) return;
|
||||
runtimeState.lastFiredAt = now;
|
||||
triggerRuntime.set(trigger.runtimeKey, runtimeState);
|
||||
events.emit('trigger', {
|
||||
buttonId: trigger.action,
|
||||
entityId: trigger.entityId,
|
||||
action: trigger.action,
|
||||
state: raw?.state ?? null,
|
||||
attributes: raw?.attributes || {},
|
||||
lastChanged: raw?.last_changed || null,
|
||||
lastUpdated: raw?.last_updated || null,
|
||||
firedAt: now,
|
||||
});
|
||||
publishEvent({
|
||||
source: 'homeAssistant',
|
||||
type: HA_BUTTON_EVENT_TYPE,
|
||||
payload: {
|
||||
buttonId: trigger.action,
|
||||
entityId: trigger.entityId,
|
||||
action: trigger.action,
|
||||
state: raw?.state ?? null,
|
||||
attributes: raw?.attributes || {},
|
||||
lastChanged: raw?.last_changed || null,
|
||||
lastUpdated: raw?.last_updated || null,
|
||||
firedAt: now,
|
||||
...(trigger.payload || {}),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function teardownConnection() {
|
||||
@@ -270,6 +386,7 @@ function getState() {
|
||||
}
|
||||
|
||||
loadEntityConfig();
|
||||
loadTriggerConfig();
|
||||
connect();
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
const sharp = require('sharp');
|
||||
const logger = require('../globals/logger').child('humanAlertButton');
|
||||
const { subscribe, publishEvent } = require('./eventBus');
|
||||
const { getMode, MODES } = require('./modeManager');
|
||||
const { getRoomCameras } = require('./roomCameraService');
|
||||
const { getRoomCameraState } = require('./roomCameraSnapshotService');
|
||||
|
||||
const HA_BUTTON_EVENT_TYPE = 'ha.button.action';
|
||||
const HUMAN_ALERT_ACTION = 'humanAlert';
|
||||
const HUMAN_ALERT_MESSAGE = 'Human alert button pressed.';
|
||||
const TILE_WIDTH = 480;
|
||||
const TILE_HEIGHT = 270;
|
||||
|
||||
logger.info('Human alert button service enabled', { action: HUMAN_ALERT_ACTION });
|
||||
|
||||
function isModeAllowed() {
|
||||
const mode = getMode();
|
||||
return mode !== MODES.ADMIN && mode !== MODES.LOCKDOWN;
|
||||
}
|
||||
|
||||
async function buildTile(camera, state) {
|
||||
const base = sharp({
|
||||
create: {
|
||||
width: TILE_WIDTH,
|
||||
height: TILE_HEIGHT,
|
||||
channels: 3,
|
||||
background: state?.frame ? '#000000' : '#222222',
|
||||
},
|
||||
});
|
||||
if (!state?.frame) {
|
||||
return base.jpeg({ quality: 75 }).toBuffer();
|
||||
}
|
||||
try {
|
||||
const frame = await sharp(state.frame)
|
||||
.resize(TILE_WIDTH, TILE_HEIGHT, {
|
||||
fit: 'cover',
|
||||
})
|
||||
.jpeg({ quality: 78 })
|
||||
.toBuffer();
|
||||
return frame;
|
||||
} catch (err) {
|
||||
logger.warn('Failed to build camera tile from frame', { cameraId: camera.id, error: err.message });
|
||||
return base.jpeg({ quality: 75 }).toBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
async function buildHorizontalMosaic() {
|
||||
const cameras = getRoomCameras();
|
||||
if (!cameras.length) return null;
|
||||
const width = TILE_WIDTH * cameras.length;
|
||||
const height = TILE_HEIGHT;
|
||||
const layers = [];
|
||||
for (let idx = 0; idx < cameras.length; idx += 1) {
|
||||
const cam = cameras[idx];
|
||||
const state = getRoomCameraState(cam.id);
|
||||
const input = await buildTile(cam, state);
|
||||
layers.push({
|
||||
input,
|
||||
left: idx * TILE_WIDTH,
|
||||
top: 0,
|
||||
});
|
||||
}
|
||||
return sharp({
|
||||
create: {
|
||||
width,
|
||||
height,
|
||||
channels: 3,
|
||||
background: '#000000',
|
||||
},
|
||||
})
|
||||
.composite(layers)
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function handleTrigger(event = {}) {
|
||||
if (String(event?.payload?.action || '') !== HUMAN_ALERT_ACTION) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
if (!isModeAllowed()) {
|
||||
logger.info('Ignoring human alert trigger due to mode gate', { mode: getMode() });
|
||||
return;
|
||||
}
|
||||
let mosaic = null;
|
||||
try {
|
||||
mosaic = await buildHorizontalMosaic();
|
||||
} catch (err) {
|
||||
logger.warn('Failed to build human alert mosaic', { error: err.message });
|
||||
}
|
||||
publishEvent({
|
||||
source: 'humanAlertButton',
|
||||
type: 'humanAlert.buttonPressed',
|
||||
payload: {
|
||||
message: HUMAN_ALERT_MESSAGE,
|
||||
triggeredAt: now,
|
||||
imageBase64: mosaic ? mosaic.toString('base64') : null,
|
||||
trigger: event?.payload || null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
subscribe(HA_BUTTON_EVENT_TYPE, (event) => {
|
||||
handleTrigger(event).catch((err) => {
|
||||
logger.warn('Failed handling human alert trigger', { error: err.message });
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = {};
|
||||
@@ -1,507 +0,0 @@
|
||||
const path = require('path');
|
||||
const { spawn } = require('child_process');
|
||||
const readline = require('readline');
|
||||
const logger = require('../globals/logger').child('roomHumanDetection');
|
||||
const io = require('../globals/io');
|
||||
const { loadConfig } = require('../helpers/configLoader');
|
||||
const { roomCameraStreamEvents } = require('./roomCameraSnapshotService');
|
||||
const roverManager = require('./roverManager');
|
||||
const { issueCommand } = require('./commandService');
|
||||
const { publishEvent } = require('./eventBus');
|
||||
const { sendAlert } = require('./alertService');
|
||||
const { getMode, MODES } = require('./modeManager');
|
||||
const { isAdmin } = require('./roleService');
|
||||
|
||||
const config = loadConfig();
|
||||
const visionConfig = config.vision?.humanDetection || {};
|
||||
|
||||
const runtime = {
|
||||
enabled: Boolean(visionConfig.enabled),
|
||||
confidenceThreshold: Number.isFinite(Number(visionConfig.confidenceThreshold))
|
||||
? Number(visionConfig.confidenceThreshold)
|
||||
: 0.55,
|
||||
ttsDelayMs: Number.isFinite(Number(visionConfig.ttsDelayMs)) ? Number(visionConfig.ttsDelayMs) : 30000,
|
||||
discordDelayMs: Number.isFinite(Number(visionConfig.discordDelayMs)) ? Number(visionConfig.discordDelayMs) : 60000,
|
||||
clearWindowMs: Number.isFinite(Number(visionConfig.clearWindowMs)) ? Number(visionConfig.clearWindowMs) : 3000,
|
||||
cooldownMs: Number.isFinite(Number(visionConfig.cooldownMs)) ? Number(visionConfig.cooldownMs) : 15 * 60 * 1000,
|
||||
maxInferenceFpsPerCamera: Number.isFinite(Number(visionConfig.maxInferenceFpsPerCamera))
|
||||
? Math.max(0.25, Number(visionConfig.maxInferenceFpsPerCamera))
|
||||
: 3,
|
||||
};
|
||||
|
||||
const TTS_TEXT = 'human detected in room, alerting soon';
|
||||
const workerScript = path.join(__dirname, '..', '..', 'scripts', 'human_detector_worker.py');
|
||||
const pythonCandidates = [
|
||||
process.env.VISION_PYTHON,
|
||||
'/opt/multiroomba-vision/.venv/bin/python3',
|
||||
'python3',
|
||||
].filter(Boolean);
|
||||
|
||||
const cameraState = new Map(); // cameraId -> { inflight,lastInferAt,lastResultAt,lastPositiveAt,lastConfidence,error }
|
||||
const history = []; // up to 80 recent events
|
||||
|
||||
let worker = null;
|
||||
let workerReady = false;
|
||||
let workerPython = null;
|
||||
let workerRestartCount = 0;
|
||||
let reqSeq = 1;
|
||||
|
||||
let lastAnyPositiveAt = null;
|
||||
let episodeStartAt = null;
|
||||
let ttsSentThisEpisode = false;
|
||||
let discordSentThisEpisode = false;
|
||||
let latestPositiveFrame = null; // { cameraId, confidence, ts, buffer }
|
||||
let cooldownUntil = 0;
|
||||
let hasClearedSinceLastDiscord = true;
|
||||
let lastDiscordAlertAt = null;
|
||||
let lastDiscordAlertMeta = null;
|
||||
|
||||
function getInferenceIntervalMs() {
|
||||
return Math.max(100, Math.round(1000 / Math.max(0.25, Number(runtime.maxInferenceFpsPerCamera) || 3)));
|
||||
}
|
||||
|
||||
function pushHistory(type, detail = {}) {
|
||||
history.push({ ts: Date.now(), type, detail });
|
||||
while (history.length > 80) history.shift();
|
||||
}
|
||||
|
||||
function isDetectionActiveMode() {
|
||||
const mode = getMode();
|
||||
return mode !== MODES.ADMIN && mode !== MODES.LOCKDOWN;
|
||||
}
|
||||
|
||||
function ensureCameraState(cameraId) {
|
||||
if (!cameraState.has(cameraId)) {
|
||||
cameraState.set(cameraId, {
|
||||
inflight: false,
|
||||
lastInferAt: 0,
|
||||
lastResultAt: 0,
|
||||
lastPositiveAt: 0,
|
||||
lastConfidence: 0,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
return cameraState.get(cameraId);
|
||||
}
|
||||
|
||||
function markCleared(now, reason = 'clear_window') {
|
||||
if (episodeStartAt == null) return;
|
||||
episodeStartAt = null;
|
||||
ttsSentThisEpisode = false;
|
||||
discordSentThisEpisode = false;
|
||||
lastAnyPositiveAt = null;
|
||||
latestPositiveFrame = null;
|
||||
hasClearedSinceLastDiscord = true;
|
||||
pushHistory('episode.cleared', { reason });
|
||||
logger.info('Human detection episode cleared', { now, reason });
|
||||
}
|
||||
|
||||
function sendTtsToNonPrivateRovers() {
|
||||
const roster = roverManager.getRoster();
|
||||
let sent = 0;
|
||||
roster.forEach((entry) => {
|
||||
if (entry?.private?.enabled) return;
|
||||
try {
|
||||
issueCommand(String(entry.id), {
|
||||
type: 'tts',
|
||||
tts: {
|
||||
text: TTS_TEXT,
|
||||
speak: true,
|
||||
},
|
||||
});
|
||||
sent += 1;
|
||||
} catch (err) {
|
||||
logger.warn('Failed to send human-alert TTS', { roverId: entry?.id, error: err.message });
|
||||
}
|
||||
});
|
||||
sendAlert({
|
||||
color: '#f0b651',
|
||||
title: 'Human Detection',
|
||||
message:
|
||||
sent > 0
|
||||
? `Person detected; announced on ${sent} rover(s).`
|
||||
: 'Person detected; no non-private rover available.',
|
||||
});
|
||||
publishEvent({
|
||||
source: 'roomHumanDetection',
|
||||
type: 'vision.humanTtsSent',
|
||||
payload: {
|
||||
text: TTS_TEXT,
|
||||
roverCount: sent,
|
||||
ts: Date.now(),
|
||||
},
|
||||
});
|
||||
pushHistory('tts.sent', { roverCount: sent });
|
||||
}
|
||||
|
||||
function sendDiscordDetectionAlert(now, options = {}) {
|
||||
const payload = {
|
||||
message: options.message || 'Human detected in room cameras.',
|
||||
detectedAt: now,
|
||||
confidence: latestPositiveFrame?.confidence || null,
|
||||
cameraId: latestPositiveFrame?.cameraId || null,
|
||||
imageBase64: latestPositiveFrame?.buffer ? latestPositiveFrame.buffer.toString('base64') : null,
|
||||
};
|
||||
publishEvent({
|
||||
source: 'roomHumanDetection',
|
||||
type: 'vision.humanDetected',
|
||||
payload,
|
||||
});
|
||||
sendAlert({
|
||||
color: '#e53935',
|
||||
title: 'Human Detection',
|
||||
message: 'Human presence persisted; Discord alert sent.',
|
||||
});
|
||||
lastDiscordAlertAt = now;
|
||||
lastDiscordAlertMeta = {
|
||||
cameraId: payload.cameraId,
|
||||
confidence: payload.confidence,
|
||||
detectedAt: payload.detectedAt,
|
||||
};
|
||||
pushHistory('discord.sent', {
|
||||
cameraId: payload.cameraId,
|
||||
confidence: payload.confidence,
|
||||
});
|
||||
}
|
||||
|
||||
function buildState(now = Date.now()) {
|
||||
const mode = getMode();
|
||||
const modeActive = isDetectionActiveMode();
|
||||
const humanPresent = episodeStartAt != null;
|
||||
const elapsed = humanPresent ? Math.max(0, now - episodeStartAt) : 0;
|
||||
const timeToTtsMs = humanPresent && !ttsSentThisEpisode ? Math.max(0, runtime.ttsDelayMs - elapsed) : 0;
|
||||
const timeToDiscordMs = humanPresent && !discordSentThisEpisode ? Math.max(0, runtime.discordDelayMs - elapsed) : 0;
|
||||
const cooldownRemainingMs = Math.max(0, cooldownUntil - now);
|
||||
const cameras = Array.from(cameraState.entries()).map(([cameraId, state]) => ({
|
||||
cameraId,
|
||||
inflight: Boolean(state.inflight),
|
||||
lastInferAt: state.lastInferAt || null,
|
||||
lastResultAt: state.lastResultAt || null,
|
||||
lastPositiveAt: state.lastPositiveAt || null,
|
||||
lastConfidence: state.lastConfidence || 0,
|
||||
error: state.error || null,
|
||||
}));
|
||||
cameras.sort((a, b) => String(a.cameraId).localeCompare(String(b.cameraId)));
|
||||
return {
|
||||
enabled: runtime.enabled,
|
||||
mode,
|
||||
modeActive,
|
||||
workerReady,
|
||||
workerRunning: Boolean(worker && !worker.killed),
|
||||
workerPython,
|
||||
workerScript,
|
||||
workerRestartCount,
|
||||
config: { ...runtime },
|
||||
episode: {
|
||||
humanPresent,
|
||||
startAt: episodeStartAt,
|
||||
lastAnyPositiveAt,
|
||||
ttsSentThisEpisode,
|
||||
discordSentThisEpisode,
|
||||
timeToTtsMs,
|
||||
timeToDiscordMs,
|
||||
cooldownUntil,
|
||||
cooldownRemainingMs,
|
||||
hasClearedSinceLastDiscord,
|
||||
},
|
||||
latestPositive: latestPositiveFrame
|
||||
? {
|
||||
cameraId: latestPositiveFrame.cameraId,
|
||||
confidence: latestPositiveFrame.confidence,
|
||||
ts: latestPositiveFrame.ts,
|
||||
}
|
||||
: null,
|
||||
lastDiscordAlertAt,
|
||||
lastDiscordAlertMeta,
|
||||
cameras,
|
||||
history: history.slice(),
|
||||
updatedAt: now,
|
||||
};
|
||||
}
|
||||
|
||||
function emitStateToAdmins() {
|
||||
const payload = buildState();
|
||||
io.sockets.sockets.forEach((socket) => {
|
||||
if (!isAdmin(socket)) return;
|
||||
socket.emit('vision:human:state', payload);
|
||||
});
|
||||
}
|
||||
|
||||
function evaluateEpisode(now = Date.now()) {
|
||||
if (!runtime.enabled) {
|
||||
markCleared(now, 'disabled');
|
||||
return;
|
||||
}
|
||||
if (!isDetectionActiveMode()) {
|
||||
markCleared(now, 'mode_gate');
|
||||
return;
|
||||
}
|
||||
if (episodeStartAt != null && lastAnyPositiveAt != null && now - lastAnyPositiveAt > runtime.clearWindowMs) {
|
||||
markCleared(now, 'clear_window');
|
||||
return;
|
||||
}
|
||||
if (episodeStartAt == null) return;
|
||||
const elapsed = Math.max(0, now - episodeStartAt);
|
||||
if (!ttsSentThisEpisode && elapsed >= runtime.ttsDelayMs) {
|
||||
sendTtsToNonPrivateRovers();
|
||||
ttsSentThisEpisode = true;
|
||||
}
|
||||
if (discordSentThisEpisode) return;
|
||||
if (elapsed < runtime.discordDelayMs) return;
|
||||
if (!hasClearedSinceLastDiscord) return;
|
||||
if (now < cooldownUntil) return;
|
||||
sendDiscordDetectionAlert(now);
|
||||
discordSentThisEpisode = true;
|
||||
hasClearedSinceLastDiscord = false;
|
||||
cooldownUntil = now + runtime.cooldownMs;
|
||||
}
|
||||
|
||||
function handleWorkerMessage(line) {
|
||||
let msg;
|
||||
try {
|
||||
msg = JSON.parse(line);
|
||||
} catch (err) {
|
||||
logger.warn('Invalid vision worker JSON', { error: err.message });
|
||||
return;
|
||||
}
|
||||
if (msg?.type === 'ready') {
|
||||
workerReady = true;
|
||||
pushHistory('worker.ready');
|
||||
logger.info('Vision worker ready');
|
||||
emitStateToAdmins();
|
||||
return;
|
||||
}
|
||||
const cameraId = String(msg?.cameraId || '');
|
||||
if (!cameraId) return;
|
||||
const state = ensureCameraState(cameraId);
|
||||
state.inflight = false;
|
||||
state.lastResultAt = Date.now();
|
||||
if (!msg.ok) {
|
||||
state.error = msg.error || 'worker error';
|
||||
logger.warn('Vision inference failed', { cameraId, error: state.error });
|
||||
evaluateEpisode(Date.now());
|
||||
emitStateToAdmins();
|
||||
return;
|
||||
}
|
||||
state.error = null;
|
||||
state.lastConfidence = Number(msg.bestConfidence || 0);
|
||||
if (msg.personDetected) {
|
||||
const now = Number(msg.ts) || Date.now();
|
||||
state.lastPositiveAt = now;
|
||||
lastAnyPositiveAt = now;
|
||||
if (episodeStartAt == null) {
|
||||
episodeStartAt = now;
|
||||
ttsSentThisEpisode = false;
|
||||
discordSentThisEpisode = false;
|
||||
pushHistory('episode.started', { cameraId });
|
||||
logger.info('Human detection episode started', { cameraId, now });
|
||||
}
|
||||
let buffer = null;
|
||||
try {
|
||||
if (msg.annotatedBase64) {
|
||||
buffer = Buffer.from(String(msg.annotatedBase64), 'base64');
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to decode annotated frame from worker', { cameraId, error: err.message });
|
||||
}
|
||||
if (buffer) {
|
||||
latestPositiveFrame = {
|
||||
cameraId,
|
||||
confidence: state.lastConfidence,
|
||||
ts: now,
|
||||
buffer,
|
||||
};
|
||||
}
|
||||
}
|
||||
evaluateEpisode(Number(msg.ts) || Date.now());
|
||||
emitStateToAdmins();
|
||||
}
|
||||
|
||||
function startWorker() {
|
||||
for (const pythonBin of pythonCandidates) {
|
||||
try {
|
||||
const proc = spawn(pythonBin, [workerScript], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
worker = proc;
|
||||
workerReady = false;
|
||||
workerPython = pythonBin;
|
||||
readline.createInterface({ input: proc.stdout }).on('line', handleWorkerMessage);
|
||||
proc.stderr.on('data', (chunk) => {
|
||||
const text = String(chunk || '').trim();
|
||||
if (!text) return;
|
||||
logger.warn('Vision worker stderr', { text: text.slice(0, 300) });
|
||||
});
|
||||
proc.on('exit', (code, signal) => {
|
||||
logger.warn('Vision worker exited', { code, signal });
|
||||
worker = null;
|
||||
workerReady = false;
|
||||
workerRestartCount += 1;
|
||||
pushHistory('worker.exit', { code, signal });
|
||||
emitStateToAdmins();
|
||||
});
|
||||
logger.info('Vision worker started', { pythonBin, workerScript });
|
||||
pushHistory('worker.started', { pythonBin });
|
||||
return true;
|
||||
} catch (err) {
|
||||
logger.warn('Failed to spawn vision worker candidate', { pythonBin, error: err.message });
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function submitFrame(cameraId, buffer, ts = Date.now()) {
|
||||
if (!runtime.enabled) return;
|
||||
if (!worker || !workerReady) return;
|
||||
const state = ensureCameraState(cameraId);
|
||||
const now = Date.now();
|
||||
if (state.inflight) return;
|
||||
if (now - state.lastInferAt < getInferenceIntervalMs()) return;
|
||||
state.inflight = true;
|
||||
state.lastInferAt = now;
|
||||
const payload = {
|
||||
reqId: `r${reqSeq++}`,
|
||||
cameraId: String(cameraId),
|
||||
ts: Number(ts) || now,
|
||||
confidenceThreshold: runtime.confidenceThreshold,
|
||||
imageBase64: buffer.toString('base64'),
|
||||
};
|
||||
try {
|
||||
worker.stdin.write(`${JSON.stringify(payload)}\n`);
|
||||
} catch (err) {
|
||||
state.inflight = false;
|
||||
state.error = err.message;
|
||||
logger.warn('Failed writing frame to vision worker', { cameraId, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
function clampNumber(value, min, max, fallback) {
|
||||
const num = Number(value);
|
||||
if (!Number.isFinite(num)) return fallback;
|
||||
return Math.max(min, Math.min(max, num));
|
||||
}
|
||||
|
||||
function updateRuntimeConfig(patch = {}) {
|
||||
const prevEnabled = runtime.enabled;
|
||||
runtime.enabled = typeof patch.enabled === 'boolean' ? patch.enabled : runtime.enabled;
|
||||
runtime.confidenceThreshold = clampNumber(patch.confidenceThreshold, 0, 1, runtime.confidenceThreshold);
|
||||
runtime.ttsDelayMs = clampNumber(patch.ttsDelayMs, 1000, 60 * 60 * 1000, runtime.ttsDelayMs);
|
||||
runtime.discordDelayMs = clampNumber(
|
||||
patch.discordDelayMs,
|
||||
runtime.ttsDelayMs,
|
||||
2 * 60 * 60 * 1000,
|
||||
runtime.discordDelayMs,
|
||||
);
|
||||
runtime.clearWindowMs = clampNumber(patch.clearWindowMs, 250, 60 * 1000, runtime.clearWindowMs);
|
||||
runtime.cooldownMs = clampNumber(patch.cooldownMs, 0, 12 * 60 * 60 * 1000, runtime.cooldownMs);
|
||||
runtime.maxInferenceFpsPerCamera = clampNumber(
|
||||
patch.maxInferenceFpsPerCamera,
|
||||
0.25,
|
||||
30,
|
||||
runtime.maxInferenceFpsPerCamera,
|
||||
);
|
||||
if (prevEnabled && !runtime.enabled) {
|
||||
markCleared(Date.now(), 'disabled');
|
||||
}
|
||||
pushHistory('config.updated', { patch });
|
||||
}
|
||||
|
||||
if (!startWorker()) {
|
||||
logger.error('Room human detection worker failed to start; detection unavailable until restart');
|
||||
pushHistory('worker.unavailable');
|
||||
}
|
||||
|
||||
roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => {
|
||||
if (!runtime.enabled) return;
|
||||
if (!isDetectionActiveMode()) return;
|
||||
if (!id || !buffer) return;
|
||||
submitFrame(String(id), buffer, ts);
|
||||
});
|
||||
|
||||
setInterval(() => {
|
||||
evaluateEpisode(Date.now());
|
||||
emitStateToAdmins();
|
||||
}, 1000);
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
socket.on('vision:human:getState', (_, cb = () => {}) => {
|
||||
try {
|
||||
if (!isAdmin(socket)) {
|
||||
cb({ error: 'Not authorized' });
|
||||
return;
|
||||
}
|
||||
cb({ ok: true, state: buildState() });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('vision:human:updateConfig', ({ config: patch } = {}, cb = () => {}) => {
|
||||
try {
|
||||
if (!isAdmin(socket)) {
|
||||
cb({ error: 'Not authorized' });
|
||||
return;
|
||||
}
|
||||
updateRuntimeConfig(patch || {});
|
||||
emitStateToAdmins();
|
||||
cb({ ok: true, state: buildState() });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('vision:human:testTts', (_, cb = () => {}) => {
|
||||
try {
|
||||
if (!isAdmin(socket)) {
|
||||
cb({ error: 'Not authorized' });
|
||||
return;
|
||||
}
|
||||
sendTtsToNonPrivateRovers();
|
||||
emitStateToAdmins();
|
||||
cb({ ok: true, state: buildState() });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('vision:human:testDiscord', (_, cb = () => {}) => {
|
||||
try {
|
||||
if (!isAdmin(socket)) {
|
||||
cb({ error: 'Not authorized' });
|
||||
return;
|
||||
}
|
||||
sendDiscordDetectionAlert(Date.now(), { message: 'Human detection test alert.' });
|
||||
emitStateToAdmins();
|
||||
cb({ ok: true, state: buildState() });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('vision:human:clear', (_, cb = () => {}) => {
|
||||
try {
|
||||
if (!isAdmin(socket)) {
|
||||
cb({ error: 'Not authorized' });
|
||||
return;
|
||||
}
|
||||
markCleared(Date.now(), 'manual_clear');
|
||||
emitStateToAdmins();
|
||||
cb({ ok: true, state: buildState() });
|
||||
} catch (err) {
|
||||
cb({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
if (isAdmin(socket)) {
|
||||
socket.emit('vision:human:state', buildState());
|
||||
}
|
||||
});
|
||||
|
||||
logger.info('Room human detection service started', {
|
||||
...runtime,
|
||||
workerScript,
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
getRoomHumanDetectionState: () => buildState(),
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
const io = require('../globals/io');
|
||||
const logger = require('../globals/logger').child('sessionService');
|
||||
const { getRole, isAdmin, roleEvents } = require('./roleService');
|
||||
const { getRole, roleEvents } = require('./roleService');
|
||||
const { getMode, modeEvents } = require('./modeManager');
|
||||
const roverManager = require('./roverManager');
|
||||
const { managerEvents } = roverManager;
|
||||
@@ -28,7 +28,6 @@ const { subscribe } = require('./eventBus');
|
||||
const { getSocketIp, isLocalNetwork } = require('../helpers/ipResolver');
|
||||
const { getAudioForwardState, audioForwardEvents } = require('./audioForwardService');
|
||||
const { getAudioLevels, audioLevelsEvents } = require('./audioLevelsService');
|
||||
const { getRoomHumanDetectionState } = require('./roomHumanDetectionService');
|
||||
|
||||
const config = loadConfig();
|
||||
const discordInvite = config.discord?.invite || null;
|
||||
@@ -137,7 +136,6 @@ function buildSession(socket) {
|
||||
isVerified: Boolean(socket?.data?.isVerified),
|
||||
audioForward: getAudioForwardState(),
|
||||
audioLevels: getAudioLevels(),
|
||||
visionHumanDetection: isAdmin(socket) ? getRoomHumanDetectionState() : null,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -24,12 +24,6 @@ export default function AdminPanel() {
|
||||
setAudioLevels,
|
||||
setPrivateSafety,
|
||||
llmControl,
|
||||
visionHumanState,
|
||||
getVisionHumanState,
|
||||
updateVisionHumanConfig,
|
||||
testVisionHumanTts,
|
||||
testVisionHumanDiscord,
|
||||
clearVisionHumanState,
|
||||
adminLogs,
|
||||
llmCommentaryState,
|
||||
} = useSession();
|
||||
@@ -52,7 +46,6 @@ export default function AdminPanel() {
|
||||
forwardGain: Number.isFinite(currentAudioLevels.forwardGain) ? currentAudioLevels.forwardGain : 1,
|
||||
});
|
||||
const [privateSafetyDrafts, setPrivateSafetyDrafts] = useState({});
|
||||
const [visionDraft, setVisionDraft] = useState(null);
|
||||
|
||||
const isAdmin =
|
||||
session?.role === 'admin' ||
|
||||
@@ -190,23 +183,6 @@ export default function AdminPanel() {
|
||||
});
|
||||
}, [currentAudioLevels.forwardGain, currentAudioLevels.hornGain, currentAudioLevels.ttsGain]);
|
||||
|
||||
useEffect(() => {
|
||||
const source = visionHumanState?.config;
|
||||
if (!source) return;
|
||||
setVisionDraft({
|
||||
enabled: Boolean(visionHumanState?.enabled),
|
||||
confidenceThreshold: Number.isFinite(Number(source.confidenceThreshold))
|
||||
? Number(source.confidenceThreshold)
|
||||
: 0.55,
|
||||
ttsDelayMs: Number.isFinite(Number(source.ttsDelayMs)) ? Number(source.ttsDelayMs) : 30000,
|
||||
discordDelayMs: Number.isFinite(Number(source.discordDelayMs)) ? Number(source.discordDelayMs) : 60000,
|
||||
clearWindowMs: Number.isFinite(Number(source.clearWindowMs)) ? Number(source.clearWindowMs) : 3000,
|
||||
cooldownMs: Number.isFinite(Number(source.cooldownMs)) ? Number(source.cooldownMs) : 900000,
|
||||
maxInferenceFpsPerCamera: Number.isFinite(Number(source.maxInferenceFpsPerCamera))
|
||||
? Number(source.maxInferenceFpsPerCamera)
|
||||
: 3,
|
||||
});
|
||||
}, [visionHumanState]);
|
||||
|
||||
useEffect(() => {
|
||||
const next = {};
|
||||
@@ -268,54 +244,6 @@ export default function AdminPanel() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleRefreshVision = async () => {
|
||||
try {
|
||||
await getVisionHumanState();
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
}
|
||||
};
|
||||
|
||||
const handleVisionDraftChange = (key) => (event) => {
|
||||
if (!visionDraft) return;
|
||||
const isCheckbox = event.target.type === 'checkbox';
|
||||
const value = isCheckbox ? Boolean(event.target.checked) : Number(event.target.value);
|
||||
setVisionDraft((current) => ({ ...(current || {}), [key]: isCheckbox ? value : value }));
|
||||
};
|
||||
|
||||
const handleSaveVisionConfig = async () => {
|
||||
if (!visionDraft) return;
|
||||
try {
|
||||
await updateVisionHumanConfig(visionDraft);
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
}
|
||||
};
|
||||
|
||||
const handleVisionTestTts = async () => {
|
||||
try {
|
||||
await testVisionHumanTts();
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
}
|
||||
};
|
||||
|
||||
const handleVisionTestDiscord = async () => {
|
||||
try {
|
||||
await testVisionHumanDiscord();
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
}
|
||||
};
|
||||
|
||||
const handleVisionClear = async () => {
|
||||
try {
|
||||
await clearVisionHumanState();
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
}
|
||||
};
|
||||
|
||||
if (!isAdmin) return null;
|
||||
|
||||
return (
|
||||
@@ -535,16 +463,6 @@ export default function AdminPanel() {
|
||||
)}
|
||||
/>
|
||||
<ReplaySnapshotHealth health={health} roster={roster} />
|
||||
<VisionHumanPanel
|
||||
state={visionHumanState}
|
||||
draft={visionDraft}
|
||||
onDraftChange={handleVisionDraftChange}
|
||||
onRefresh={handleRefreshVision}
|
||||
onSaveConfig={handleSaveVisionConfig}
|
||||
onTestTts={handleVisionTestTts}
|
||||
onTestDiscord={handleVisionTestDiscord}
|
||||
onClear={handleVisionClear}
|
||||
/>
|
||||
<LlmCommentaryPanel
|
||||
state={llmCommentaryState}
|
||||
onClearHistory={handleClearLlmHistory}
|
||||
@@ -555,189 +473,6 @@ export default function AdminPanel() {
|
||||
);
|
||||
}
|
||||
|
||||
function VisionHumanPanel({
|
||||
state,
|
||||
draft,
|
||||
onDraftChange,
|
||||
onRefresh,
|
||||
onSaveConfig,
|
||||
onTestTts,
|
||||
onTestDiscord,
|
||||
onClear,
|
||||
}) {
|
||||
if (!state) {
|
||||
return (
|
||||
<div className="space-y-0.5">
|
||||
<div className="panel-muted text-xs uppercase">Human Detection</div>
|
||||
<div className="surface text-xs text-slate-300">No human detection state received yet.</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const modeGateText = state.modeActive ? 'active' : `inactive in ${state.mode}`;
|
||||
const workerText = state.workerReady ? 'ready' : state.workerRunning ? 'starting' : 'offline';
|
||||
const episode = state.episode || {};
|
||||
const history = Array.isArray(state.history) ? state.history : [];
|
||||
const cameras = Array.isArray(state.cameras) ? state.cameras : [];
|
||||
const statusPills = [
|
||||
{ label: 'enabled', value: state.enabled ? 'yes' : 'no' },
|
||||
{ label: 'mode gate', value: modeGateText },
|
||||
{ label: 'worker', value: workerText },
|
||||
{ label: 'python', value: state.workerPython || '--' },
|
||||
{ label: 'worker restarts', value: state.workerRestartCount ?? 0 },
|
||||
{ label: 'present', value: episode.humanPresent ? 'yes' : 'no' },
|
||||
{ label: 'tts sent', value: episode.ttsSentThisEpisode ? 'yes' : 'no' },
|
||||
{ label: 'discord sent', value: episode.discordSentThisEpisode ? 'yes' : 'no' },
|
||||
{ label: 'tts in', value: `${Math.ceil((episode.timeToTtsMs || 0) / 1000)}s` },
|
||||
{ label: 'discord in', value: `${Math.ceil((episode.timeToDiscordMs || 0) / 1000)}s` },
|
||||
{ label: 'cooldown', value: `${Math.ceil((episode.cooldownRemainingMs || 0) / 1000)}s` },
|
||||
];
|
||||
return (
|
||||
<div className="space-y-0.5">
|
||||
<div className="panel-muted text-xs uppercase">Human Detection</div>
|
||||
<div className="flex flex-wrap gap-0.5 text-xs">
|
||||
<button type="button" onClick={onRefresh} className="button-dark">
|
||||
Refresh
|
||||
</button>
|
||||
<button type="button" onClick={onSaveConfig} className="button-dark">
|
||||
Save Detection Config
|
||||
</button>
|
||||
<button type="button" onClick={onTestTts} className="button-dark">
|
||||
Test TTS
|
||||
</button>
|
||||
<button type="button" onClick={onTestDiscord} className="button-dark">
|
||||
Test Discord
|
||||
</button>
|
||||
<button type="button" onClick={onClear} className="button-danger">
|
||||
Clear Episode
|
||||
</button>
|
||||
</div>
|
||||
<div className="surface flex flex-wrap gap-0.5 text-xs">
|
||||
{statusPills.map((pill) => (
|
||||
<span
|
||||
key={pill.label}
|
||||
className="rounded border border-slate-600/60 bg-slate-800/70 px-0.5 py-0.25 text-[0.72rem] leading-tight text-slate-200"
|
||||
>
|
||||
{pill.label}: {pill.value}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
{draft ? (
|
||||
<div className="grid gap-0.5 md:grid-cols-2">
|
||||
<label className="surface flex items-center justify-between gap-0.5 text-xs">
|
||||
<span>Enabled</span>
|
||||
<input type="checkbox" checked={Boolean(draft.enabled)} onChange={onDraftChange('enabled')} />
|
||||
</label>
|
||||
<label className="surface grid gap-0.25 text-xs">
|
||||
<span>Confidence threshold</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="1"
|
||||
step="0.01"
|
||||
value={draft.confidenceThreshold}
|
||||
onChange={onDraftChange('confidenceThreshold')}
|
||||
className="field-input text-xs"
|
||||
/>
|
||||
</label>
|
||||
<label className="surface grid gap-0.25 text-xs">
|
||||
<span>TTS delay (ms)</span>
|
||||
<input
|
||||
type="number"
|
||||
min="1000"
|
||||
step="250"
|
||||
value={draft.ttsDelayMs}
|
||||
onChange={onDraftChange('ttsDelayMs')}
|
||||
className="field-input text-xs"
|
||||
/>
|
||||
</label>
|
||||
<label className="surface grid gap-0.25 text-xs">
|
||||
<span>Discord delay (ms)</span>
|
||||
<input
|
||||
type="number"
|
||||
min="1000"
|
||||
step="250"
|
||||
value={draft.discordDelayMs}
|
||||
onChange={onDraftChange('discordDelayMs')}
|
||||
className="field-input text-xs"
|
||||
/>
|
||||
</label>
|
||||
<label className="surface grid gap-0.25 text-xs">
|
||||
<span>Clear window (ms)</span>
|
||||
<input
|
||||
type="number"
|
||||
min="250"
|
||||
step="250"
|
||||
value={draft.clearWindowMs}
|
||||
onChange={onDraftChange('clearWindowMs')}
|
||||
className="field-input text-xs"
|
||||
/>
|
||||
</label>
|
||||
<label className="surface grid gap-0.25 text-xs">
|
||||
<span>Cooldown (ms)</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
step="1000"
|
||||
value={draft.cooldownMs}
|
||||
onChange={onDraftChange('cooldownMs')}
|
||||
className="field-input text-xs"
|
||||
/>
|
||||
</label>
|
||||
<label className="surface grid gap-0.25 text-xs">
|
||||
<span>Max inference FPS/camera</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0.25"
|
||||
max="30"
|
||||
step="0.25"
|
||||
value={draft.maxInferenceFpsPerCamera}
|
||||
onChange={onDraftChange('maxInferenceFpsPerCamera')}
|
||||
className="field-input text-xs"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
) : null}
|
||||
<div className="space-y-0.5">
|
||||
<div className="panel-muted text-xs uppercase">Per Camera</div>
|
||||
<div className="surface max-h-40 overflow-y-auto text-xs text-slate-200">
|
||||
{cameras.length ? (
|
||||
cameras.map((cam) => (
|
||||
<div key={cam.cameraId} className="flex items-center justify-between gap-0.5">
|
||||
<span>{cam.cameraId}</span>
|
||||
<span>{cam.lastConfidence?.toFixed?.(2) ?? '0.00'}</span>
|
||||
<span>{cam.lastPositiveAt ? new Date(cam.lastPositiveAt).toLocaleTimeString() : '--'}</span>
|
||||
<span className={cam.error ? 'text-amber-300' : 'text-emerald-300'}>
|
||||
{cam.error ? 'error' : cam.inflight ? 'inference' : 'ok'}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<div className="text-slate-400">No camera state yet.</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="space-y-0.5">
|
||||
<div className="panel-muted text-xs uppercase">Recent Events</div>
|
||||
<div className="surface max-h-40 overflow-y-auto text-xs text-slate-200">
|
||||
{history.length ? (
|
||||
history
|
||||
.slice()
|
||||
.reverse()
|
||||
.map((entry, idx) => (
|
||||
<div key={`${entry.ts}-${entry.type}-${idx}`} className="flex items-start justify-between gap-0.5">
|
||||
<span>{entry.type}</span>
|
||||
<span className="text-slate-400">{entry.ts ? new Date(entry.ts).toLocaleTimeString() : '--'}</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<div className="text-slate-400">No events yet.</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function LlmCommentaryPanel({ state, onClearHistory, clearingHistory }) {
|
||||
const [selectedRunId, setSelectedRunId] = useState(null);
|
||||
if (!state) {
|
||||
|
||||
@@ -10,7 +10,6 @@ const SessionContext = createContext({
|
||||
adminLogs: [],
|
||||
llmCommentaryState: null,
|
||||
llmCommentaryStatus: null,
|
||||
visionHumanState: null,
|
||||
identifySession: async () => {},
|
||||
login: async () => {},
|
||||
setRole: async () => {},
|
||||
@@ -36,11 +35,6 @@ const SessionContext = createContext({
|
||||
setAudioLevels: async () => {},
|
||||
setPrivateSafety: async () => {},
|
||||
llmControl: async () => {},
|
||||
getVisionHumanState: async () => {},
|
||||
updateVisionHumanConfig: async () => {},
|
||||
testVisionHumanTts: async () => {},
|
||||
testVisionHumanDiscord: async () => {},
|
||||
clearVisionHumanState: async () => {},
|
||||
});
|
||||
|
||||
function useAckEmitter(socket) {
|
||||
@@ -67,7 +61,6 @@ export function SessionProvider({ children }) {
|
||||
const [adminLogs, setAdminLogs] = useState([]);
|
||||
const [llmCommentaryState, setLlmCommentaryState] = useState(null);
|
||||
const [llmCommentaryStatus, setLlmCommentaryStatus] = useState(null);
|
||||
const [visionHumanState, setVisionHumanState] = useState(null);
|
||||
const [alerts, setAlerts] = useState([]);
|
||||
const [connected, setConnected] = useState(socket.connected);
|
||||
|
||||
@@ -85,9 +78,6 @@ export function SessionProvider({ children }) {
|
||||
useEffect(() => {
|
||||
function handleSession(payload) {
|
||||
setSession(payload);
|
||||
if (payload?.visionHumanDetection) {
|
||||
setVisionHumanState(payload.visionHumanDetection);
|
||||
}
|
||||
}
|
||||
function handleLogInit(entries = []) {
|
||||
setLogs(entries);
|
||||
@@ -114,10 +104,6 @@ export function SessionProvider({ children }) {
|
||||
socket.on('adminlog:init', handleAdminLogInit);
|
||||
socket.on('adminlog:entry', handleAdminLogEntry);
|
||||
socket.on('llm:state', handleLlmState);
|
||||
socket.on('vision:human:state', (payload = null) => {
|
||||
const state = payload && typeof payload === 'object' ? payload : null;
|
||||
setVisionHumanState(state);
|
||||
});
|
||||
socket.on('alert:new', (payload = {}) => {
|
||||
setAlerts((prev) => [
|
||||
...prev.slice(-49),
|
||||
@@ -134,7 +120,6 @@ export function SessionProvider({ children }) {
|
||||
socket.off('adminlog:init', handleAdminLogInit);
|
||||
socket.off('adminlog:entry', handleAdminLogEntry);
|
||||
socket.off('llm:state', handleLlmState);
|
||||
socket.off('vision:human:state');
|
||||
socket.off('alert:new');
|
||||
};
|
||||
}, [socket]);
|
||||
@@ -177,12 +162,6 @@ export function SessionProvider({ children }) {
|
||||
emitWithAck('session:privateSafety:set', { roverId, safety }),
|
||||
llmControl: (action, controls = {}) =>
|
||||
emitWithAck('llm:control', { controls: { action, ...controls } }),
|
||||
getVisionHumanState: () => emitWithAck('vision:human:getState'),
|
||||
updateVisionHumanConfig: (nextConfig = {}) =>
|
||||
emitWithAck('vision:human:updateConfig', { config: nextConfig }),
|
||||
testVisionHumanTts: () => emitWithAck('vision:human:testTts'),
|
||||
testVisionHumanDiscord: () => emitWithAck('vision:human:testDiscord'),
|
||||
clearVisionHumanState: () => emitWithAck('vision:human:clear'),
|
||||
pushAlert: (alert) =>
|
||||
setAlerts((prev) => [
|
||||
...prev.slice(-49),
|
||||
@@ -200,21 +179,10 @@ export function SessionProvider({ children }) {
|
||||
adminLogs,
|
||||
llmCommentaryState,
|
||||
llmCommentaryStatus,
|
||||
visionHumanState,
|
||||
alerts,
|
||||
...actions,
|
||||
}),
|
||||
[
|
||||
actions,
|
||||
adminLogs,
|
||||
alerts,
|
||||
connected,
|
||||
llmCommentaryState,
|
||||
llmCommentaryStatus,
|
||||
logs,
|
||||
session,
|
||||
visionHumanState,
|
||||
],
|
||||
[actions, adminLogs, alerts, connected, llmCommentaryState, llmCommentaryStatus, logs, session],
|
||||
);
|
||||
|
||||
return <SessionContext.Provider value={value}>{children}</SessionContext.Provider>;
|
||||
|
||||
Reference in New Issue
Block a user