From 1e9a6b2e19b31754b38672f9ea23a93a1fbb952c Mon Sep 17 00:00:00 2001 From: legop3 Date: Mon, 6 Apr 2026 00:12:00 -0400 Subject: [PATCH] opencv --- plans/to-do.md | 2 +- server/config.example.yaml | 17 + server/index.js | 1 + server/install_server.sh | 30 +- server/scripts/human_detector_worker.py | 135 ++++++++ server/src/services/discordBotService.js | 40 ++- .../src/services/roomHumanDetectionService.js | 291 ++++++++++++++++++ 7 files changed, 507 insertions(+), 9 deletions(-) create mode 100644 server/scripts/human_detector_worker.py create mode 100644 server/src/services/roomHumanDetectionService.js diff --git a/plans/to-do.md b/plans/to-do.md index ceb104cf..42c7543e 100644 --- a/plans/to-do.md +++ b/plans/to-do.md @@ -2,7 +2,7 @@ 2. trusted user system [x] 3. private rovers [x] 4. optional bump-off in drive macro [x] -5. allow admins to click on locked rovers from the roster +5. allow admins to click on locked rovers from the roster [x] 6. add faster way for admins to login 7. custom webhook profile pictures for chat bridge in discord 8. home assistant switch that tells the server to force the lights on diff --git a/server/config.example.yaml b/server/config.example.yaml index aab99d12..0282724a 100644 --- a/server/config.example.yaml +++ b/server/config.example.yaml @@ -52,6 +52,21 @@ roomCameras: url: "http://192.168.0.51/snapshot.jpg" streamUrl: "http://192.168.0.51/stream.mjpg" +vision: + humanDetection: + enabled: false + confidenceThreshold: 0.55 + # Continuous presence windows: + # - 30s => rover TTS warning + # - 60s => Discord human alert + ttsDelayMs: 30000 + discordDelayMs: 60000 + # Tolerates brief detector misses before declaring "cleared". + clearWindowMs: 3000 + # After a Discord alert, require clear + cooldown before next alert. + cooldownMs: 900000 + maxInferenceFpsPerCamera: 3 + discord: token: "DISCORD_BOT_TOKEN" guildId: "123456789012345678" # optional; bot works in any guild it's invited to @@ -61,9 +76,11 @@ discord: adminAlerts: "123456789012345678" # chat bridge is configured per guild via `rs bridge` commands replay: "123456789012345678" + humanAlerts: "123456789012345678" roles: announcementPing: "123456789012345678" adminPing: "123456789012345678" + humanAlertPing: "123456789012345678" socials: - id: "discord" diff --git a/server/index.js b/server/index.js index 9589457d..74d57727 100644 --- a/server/index.js +++ b/server/index.js @@ -27,6 +27,7 @@ require('./src/services/videoAuthService'); require('./src/services/videoSocketService'); require('./src/services/roomCameraSocketService'); require('./src/services/roverSnapshotSocketService'); +require('./src/services/roomHumanDetectionService'); require('./src/services/embedHttpService'); require('./src/services/logStreamService'); require('./src/services/adminLogService'); diff --git a/server/install_server.sh b/server/install_server.sh index 22e547c4..9ade48f9 100755 --- a/server/install_server.sh +++ b/server/install_server.sh @@ -10,6 +10,9 @@ MEDIAMTX_SERVICE="/etc/systemd/system/mediamtx.service" MULTIROVER_SERVICE="/etc/systemd/system/multirover.service" SNAPSHOT_DIR="/var/lib/rover-snapshots" REPLAY_SEGMENT_DIR="/var/lib/replay-segments" +VISION_HOME="/opt/multiroomba-vision" +VISION_VENV="$VISION_HOME/.venv" +VISION_PYTHON="$VISION_VENV/bin/python3" if [[ $EUID -ne 0 ]]; then echo "This installer must be run with sudo/root." >&2 @@ -27,11 +30,11 @@ SERVER_DIR="$SCRIPT_DIR" CONFIG_PATH="$SERVER_DIR/config.yaml" MEDIAMTX_TEMPLATE="$SERVER_DIR/mediamtx/mediamtx.yml" -echo "[1/6] Installing dependencies..." -dnf install -y nodejs npm curl tar >/dev/null +echo "[1/7] Installing dependencies..." +dnf install -y nodejs npm curl tar python3 python3-pip >/dev/null NODE_BIN="$(command -v node)" -echo "[2/6] Installing Node production deps..." +echo "[2/7] Installing Node production deps..." runuser -u "$TARGET_USER" -- bash -c "cd '$SERVER_DIR' && npm install --production" if [[ ! -f "$CONFIG_PATH" ]]; then @@ -40,6 +43,18 @@ if [[ ! -f "$CONFIG_PATH" ]]; then echo "Copied config.example.yaml to config.yaml; edit it before exposing the service." fi +echo "[3/7] Installing OpenCV runtime..." +mkdir -p "$VISION_HOME" +python3 -m venv "$VISION_VENV" +"$VISION_PYTHON" -m pip install --upgrade pip >/dev/null +"$VISION_PYTHON" -m pip install "opencv-python-headless==4.10.0.84" "numpy==1.26.4" >/dev/null +"$VISION_PYTHON" - <<'PY' +import cv2 +import numpy +print("Vision runtime OK", cv2.__version__, numpy.__version__) +PY +chown -R "$TARGET_USER":"$TARGET_USER" "$VISION_HOME" + tmpdir=$(mktemp -d) trap 'rm -rf "$tmpdir"' EXIT @@ -60,7 +75,7 @@ case "$arch" in ;; esac -echo "[3/6] Installing mediaMTX ${MEDIAMTX_VERSION}..." +echo "[4/7] Installing mediaMTX ${MEDIAMTX_VERSION}..." curl -L "$MEDIAMTX_BASE_URL/$mediamtx_pkg" -o "$tmpdir/mediamtx.tgz" tar -xzf "$tmpdir/mediamtx.tgz" -C "$tmpdir" mediamtx install -m 0755 "$tmpdir/mediamtx" "$MEDIAMTX_BIN" @@ -75,7 +90,7 @@ rm -f "$MEDIAMTX_CONFIG" install -m 0644 "$MEDIAMTX_TEMPLATE" "$MEDIAMTX_CONFIG" chown -R "$TARGET_USER":"$TARGET_USER" "$MEDIAMTX_CONF_DIR" -echo "[4/6] Writing systemd units..." +echo "[5/7] Writing systemd units..." mkdir -p "$SNAPSHOT_DIR" chown "$TARGET_USER":"$TARGET_USER" "$SNAPSHOT_DIR" mkdir -p "$REPLAY_SEGMENT_DIR" @@ -113,6 +128,7 @@ Environment=NODE_ENV=production Environment=SERVER_CONFIG=$CONFIG_PATH Environment=ROVER_SNAPSHOT_DIR=$SNAPSHOT_DIR Environment=REPLAY_SEGMENT_DIR=$REPLAY_SEGMENT_DIR +Environment=VISION_PYTHON=$VISION_PYTHON ExecStart=$NODE_BIN $SERVER_DIR/index.js Restart=on-failure RestartSec=2 @@ -123,14 +139,14 @@ EOF chmod 644 "$MEDIAMTX_SERVICE" "$MULTIROVER_SERVICE" -echo "[5/6] Enabling services..." +echo "[6/7] Enabling services..." systemctl daemon-reload systemctl enable --now mediamtx.service systemctl enable --now multirover.service systemctl restart mediamtx.service systemctl restart multirover.service -echo "[6/6] Done." +echo "[7/7] Done." echo echo "Services installed:" echo " mediamtx.service (WebRTC fan-out)" diff --git a/server/scripts/human_detector_worker.py b/server/scripts/human_detector_worker.py new file mode 100644 index 00000000..f2d97da8 --- /dev/null +++ b/server/scripts/human_detector_worker.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +import base64 +import json +import sys +from datetime import datetime, timezone + +import cv2 +import numpy as np + + +MAX_EDGE = 640 + + +def now_iso(): + return datetime.now(timezone.utc).isoformat() + + +def emit(payload): + sys.stdout.write(json.dumps(payload, separators=(",", ":")) + "\n") + sys.stdout.flush() + + +def decode_image(image_b64): + raw = base64.b64decode(image_b64) + arr = np.frombuffer(raw, dtype=np.uint8) + frame = cv2.imdecode(arr, cv2.IMREAD_COLOR) + if frame is None: + raise ValueError("failed to decode image") + return frame + + +def resize_for_inference(frame): + h, w = frame.shape[:2] + max_side = max(h, w) + if max_side <= MAX_EDGE: + return frame + scale = float(MAX_EDGE) / float(max_side) + nw = max(1, int(round(w * scale))) + nh = max(1, int(round(h * scale))) + return cv2.resize(frame, (nw, nh), interpolation=cv2.INTER_AREA) + + +def encode_jpeg(frame): + ok, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 78]) + if not ok: + raise ValueError("failed to encode image") + return base64.b64encode(encoded.tobytes()).decode("ascii") + + +def annotate_frame(frame, camera_id, confidence): + stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + msg = f"person {confidence:.2f} | {camera_id} | {stamp}" + cv2.rectangle(frame, (8, 8), (min(frame.shape[1] - 8, 540), 42), (0, 0, 0), -1) + cv2.putText( + frame, + msg, + (14, 30), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (0, 255, 0), + 2, + cv2.LINE_AA, + ) + return frame + + +hog = cv2.HOGDescriptor() +hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector()) + +emit({"type": "ready", "ts": now_iso()}) + +for line in sys.stdin: + line = line.strip() + if not line: + continue + req = None + try: + req = json.loads(line) + req_id = req.get("reqId") + cam_id = str(req.get("cameraId", "unknown")) + conf_threshold = float(req.get("confidenceThreshold", 0.55)) + frame = decode_image(req["imageBase64"]) + working = resize_for_inference(frame) + rects, weights = hog.detectMultiScale( + working, + winStride=(8, 8), + padding=(8, 8), + scale=1.05, + ) + best = 0.0 + detections = [] + if weights is None: + weights = [] + for i, rect in enumerate(rects): + weight = float(weights[i]) if i < len(weights) else 0.0 + best = max(best, weight) + if weight < conf_threshold: + continue + x, y, w, h = rect + detections.append({"x": int(x), "y": int(y), "w": int(w), "h": int(h), "confidence": weight}) + cv2.rectangle(working, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 0), 2) + cv2.putText( + working, + f"{weight:.2f}", + (int(x), max(12, int(y) - 6)), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (0, 255, 0), + 1, + cv2.LINE_AA, + ) + detected = len(detections) > 0 + if detected: + working = annotate_frame(working, cam_id, max(d["confidence"] for d in detections)) + emit( + { + "ok": True, + "reqId": req_id, + "cameraId": cam_id, + "personDetected": detected, + "bestConfidence": best, + "detections": detections, + "annotatedBase64": encode_jpeg(working), + "ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000), + } + ) + except Exception as err: + emit( + { + "ok": False, + "reqId": req.get("reqId") if isinstance(req, dict) else None, + "error": str(err), + "ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000), + } + ) diff --git a/server/src/services/discordBotService.js b/server/src/services/discordBotService.js index 35171c5a..675c4b60 100644 --- a/server/src/services/discordBotService.js +++ b/server/src/services/discordBotService.js @@ -1231,6 +1231,7 @@ async function announce({ title, description, embeds, + files, }) { if (!channelId) return; const mentionChunks = []; @@ -1247,7 +1248,7 @@ async function announce({ await sendToChannel( channelId, `${prefix}${content || ''}`.trim(), - { embeds: payloadEmbeds }, + { embeds: payloadEmbeds, files: Array.isArray(files) ? files : undefined }, allowedMentions, !pingRoleId, // keep mention intact when pinging ); @@ -1459,6 +1460,43 @@ function handleBusEvent(event) { logger.warn('Replay send failed', err.message); }); break; + case 'vision.humanDetected': { + const imageBase64 = payload?.imageBase64 ? String(payload.imageBase64) : ''; + let attachment = null; + if (imageBase64) { + try { + const imageBuffer = Buffer.from(imageBase64, 'base64'); + if (imageBuffer.length > 0) { + attachment = new AttachmentBuilder(imageBuffer, { name: 'human-detected.jpg' }); + } + } catch (err) { + logger.warn('Failed to decode human detection image for Discord', err.message); + } + } + const cameraLabel = payload?.cameraId ? `Camera: \`${payload.cameraId}\`` : 'Camera: unknown'; + const confidenceLabel = + Number.isFinite(Number(payload?.confidence)) + ? `Confidence: \`${Number(payload.confidence).toFixed(2)}\`` + : 'Confidence: n/a'; + const detectedAt = Number(payload?.detectedAt); + const detectedLabel = Number.isFinite(detectedAt) + ? `Detected: ` + : null; + const embed = buildEmbed({ + title: 'Human Detected', + description: [cameraLabel, confidenceLabel, detectedLabel].filter(Boolean).join('\n'), + color: 0xe53935, + }); + announce({ + channelId: channels.humanAlerts, + pingRoleId: roles.humanAlertPing || null, + content: payload?.message || 'Human detected in room.', + embeds: [embed], + files: attachment ? [attachment] : [], + includeSiteUrl: false, + }); + break; + } default: break; } diff --git a/server/src/services/roomHumanDetectionService.js b/server/src/services/roomHumanDetectionService.js new file mode 100644 index 00000000..7082184f --- /dev/null +++ b/server/src/services/roomHumanDetectionService.js @@ -0,0 +1,291 @@ +const path = require('path'); +const { spawn } = require('child_process'); +const readline = require('readline'); +const logger = require('../globals/logger').child('roomHumanDetection'); +const { loadConfig } = require('../helpers/configLoader'); +const { roomCameraStreamEvents } = require('./roomCameraSnapshotService'); +const roverManager = require('./roverManager'); +const { issueCommand } = require('./commandService'); +const { publishEvent } = require('./eventBus'); +const { sendAlert } = require('./alertService'); + +const config = loadConfig(); +const visionConfig = config.vision?.humanDetection || {}; +const enabled = Boolean(visionConfig.enabled); + +if (!enabled) { + logger.info('Room human detection disabled via config'); + return; +} + +const CONFIDENCE_THRESHOLD = Number.isFinite(Number(visionConfig.confidenceThreshold)) + ? Number(visionConfig.confidenceThreshold) + : 0.55; +const TTS_DELAY_MS = Number.isFinite(Number(visionConfig.ttsDelayMs)) + ? Number(visionConfig.ttsDelayMs) + : 30000; +const DISCORD_DELAY_MS = Number.isFinite(Number(visionConfig.discordDelayMs)) + ? Number(visionConfig.discordDelayMs) + : 60000; +const CLEAR_WINDOW_MS = Number.isFinite(Number(visionConfig.clearWindowMs)) + ? Number(visionConfig.clearWindowMs) + : 3000; +const COOLDOWN_MS = Number.isFinite(Number(visionConfig.cooldownMs)) + ? Number(visionConfig.cooldownMs) + : 15 * 60 * 1000; +const MAX_INFERENCE_FPS = Number.isFinite(Number(visionConfig.maxInferenceFpsPerCamera)) + ? Math.max(0.25, Number(visionConfig.maxInferenceFpsPerCamera)) + : 3; +const TTS_TEXT = 'human detected in room, alerting soon'; + +const workerScript = path.join(__dirname, '..', '..', 'scripts', 'human_detector_worker.py'); +const pythonCandidates = [process.env.VISION_PYTHON, '/opt/multiroomba-vision/.venv/bin/python3', 'python3'].filter(Boolean); +const inferenceIntervalMs = Math.max(100, Math.round(1000 / MAX_INFERENCE_FPS)); + +const cameraState = new Map(); // cameraId -> { inflight,lastInferAt,lastResultAt,lastPositiveAt,lastConfidence,error } +let worker = null; +let workerReady = false; +let reqSeq = 1; + +let lastAnyPositiveAt = null; +let episodeStartAt = null; +let ttsSentThisEpisode = false; +let discordSentThisEpisode = false; +let latestPositiveFrame = null; // { cameraId, confidence, ts, buffer } +let cooldownUntil = 0; +let hasClearedSinceLastDiscord = true; + +function ensureCameraState(cameraId) { + if (!cameraState.has(cameraId)) { + cameraState.set(cameraId, { + inflight: false, + lastInferAt: 0, + lastResultAt: 0, + lastPositiveAt: 0, + lastConfidence: 0, + error: null, + }); + } + return cameraState.get(cameraId); +} + +function markCleared(now) { + if (episodeStartAt == null) return; + episodeStartAt = null; + ttsSentThisEpisode = false; + discordSentThisEpisode = false; + lastAnyPositiveAt = null; + latestPositiveFrame = null; + hasClearedSinceLastDiscord = true; + logger.info('Human detection episode cleared', { now }); +} + +function sendTtsToNonPrivateRovers() { + const roster = roverManager.getRoster(); + let sent = 0; + roster.forEach((entry) => { + if (entry?.private?.enabled) return; + try { + issueCommand(String(entry.id), { + type: 'tts', + tts: { + text: TTS_TEXT, + speak: true, + }, + }); + sent += 1; + } catch (err) { + logger.warn('Failed to send human-alert TTS', { roverId: entry?.id, error: err.message }); + } + }); + sendAlert({ + color: '#f0b651', + title: 'Human Detection', + message: sent > 0 ? `Person detected; announced on ${sent} rover(s).` : 'Person detected; no non-private rover available.', + }); + publishEvent({ + source: 'roomHumanDetection', + type: 'vision.humanTtsSent', + payload: { + text: TTS_TEXT, + roverCount: sent, + ts: Date.now(), + }, + }); +} + +function sendDiscordDetectionAlert(now) { + const payload = { + message: 'Human detected in room cameras.', + detectedAt: now, + confidence: latestPositiveFrame?.confidence || null, + cameraId: latestPositiveFrame?.cameraId || null, + imageBase64: latestPositiveFrame?.buffer ? latestPositiveFrame.buffer.toString('base64') : null, + }; + publishEvent({ + source: 'roomHumanDetection', + type: 'vision.humanDetected', + payload, + }); + sendAlert({ + color: '#e53935', + title: 'Human Detection', + message: 'Human presence persisted; Discord alert sent.', + }); +} + +function evaluateEpisode(now = Date.now()) { + if (episodeStartAt != null && lastAnyPositiveAt != null && now - lastAnyPositiveAt > CLEAR_WINDOW_MS) { + markCleared(now); + return; + } + if (episodeStartAt == null) return; + const elapsed = Math.max(0, now - episodeStartAt); + if (!ttsSentThisEpisode && elapsed >= TTS_DELAY_MS) { + sendTtsToNonPrivateRovers(); + ttsSentThisEpisode = true; + } + if (discordSentThisEpisode) return; + if (elapsed < DISCORD_DELAY_MS) return; + if (!hasClearedSinceLastDiscord) return; + if (now < cooldownUntil) return; + sendDiscordDetectionAlert(now); + discordSentThisEpisode = true; + hasClearedSinceLastDiscord = false; + cooldownUntil = now + COOLDOWN_MS; +} + +function handleWorkerMessage(line) { + let msg; + try { + msg = JSON.parse(line); + } catch (err) { + logger.warn('Invalid vision worker JSON', { error: err.message }); + return; + } + if (msg?.type === 'ready') { + workerReady = true; + logger.info('Vision worker ready'); + return; + } + const cameraId = String(msg?.cameraId || ''); + if (!cameraId) return; + const state = ensureCameraState(cameraId); + state.inflight = false; + state.lastResultAt = Date.now(); + if (!msg.ok) { + state.error = msg.error || 'worker error'; + logger.warn('Vision inference failed', { cameraId, error: state.error }); + evaluateEpisode(Date.now()); + return; + } + state.error = null; + state.lastConfidence = Number(msg.bestConfidence || 0); + if (msg.personDetected) { + const now = Number(msg.ts) || Date.now(); + state.lastPositiveAt = now; + lastAnyPositiveAt = now; + if (episodeStartAt == null) { + episodeStartAt = now; + ttsSentThisEpisode = false; + discordSentThisEpisode = false; + logger.info('Human detection episode started', { cameraId, now }); + } + let buffer = null; + try { + if (msg.annotatedBase64) { + buffer = Buffer.from(String(msg.annotatedBase64), 'base64'); + } + } catch (err) { + logger.warn('Failed to decode annotated frame from worker', { cameraId, error: err.message }); + } + if (buffer) { + latestPositiveFrame = { + cameraId, + confidence: state.lastConfidence, + ts: now, + buffer, + }; + } + } + evaluateEpisode(Number(msg.ts) || Date.now()); +} + +function startWorker() { + for (const pythonBin of pythonCandidates) { + try { + const proc = spawn(pythonBin, [workerScript], { + stdio: ['pipe', 'pipe', 'pipe'], + }); + worker = proc; + workerReady = false; + readline + .createInterface({ input: proc.stdout }) + .on('line', handleWorkerMessage); + proc.stderr.on('data', (chunk) => { + const text = String(chunk || '').trim(); + if (!text) return; + logger.warn('Vision worker stderr', { text: text.slice(0, 300) }); + }); + proc.on('exit', (code, signal) => { + logger.warn('Vision worker exited', { code, signal }); + worker = null; + workerReady = false; + }); + logger.info('Vision worker started', { pythonBin, workerScript }); + return true; + } catch (err) { + logger.warn('Failed to spawn vision worker candidate', { pythonBin, error: err.message }); + } + } + return false; +} + +function submitFrame(cameraId, buffer, ts = Date.now()) { + if (!worker || !workerReady) return; + const state = ensureCameraState(cameraId); + const now = Date.now(); + if (state.inflight) return; + if (now - state.lastInferAt < inferenceIntervalMs) return; + state.inflight = true; + state.lastInferAt = now; + const payload = { + reqId: `r${reqSeq++}`, + cameraId: String(cameraId), + ts: Number(ts) || now, + confidenceThreshold: CONFIDENCE_THRESHOLD, + imageBase64: buffer.toString('base64'), + }; + try { + worker.stdin.write(`${JSON.stringify(payload)}\n`); + } catch (err) { + state.inflight = false; + state.error = err.message; + logger.warn('Failed writing frame to vision worker', { cameraId, error: err.message }); + } +} + +if (!startWorker()) { + logger.error('Room human detection disabled; failed to start Python worker'); + return; +} + +roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => { + if (!id || !buffer) return; + submitFrame(String(id), buffer, ts); +}); + +setInterval(() => { + evaluateEpisode(Date.now()); +}, 1000); + +logger.info('Room human detection service started', { + confidenceThreshold: CONFIDENCE_THRESHOLD, + ttsDelayMs: TTS_DELAY_MS, + discordDelayMs: DISCORD_DELAY_MS, + clearWindowMs: CLEAR_WINDOW_MS, + cooldownMs: COOLDOWN_MS, + maxInferenceFpsPerCamera: MAX_INFERENCE_FPS, +}); + +module.exports = {};