opencv

server/config.example.yaml

@@ -52,6 +52,21 @@ roomCameras:
     url: "http://192.168.0.51/snapshot.jpg"
     streamUrl: "http://192.168.0.51/stream.mjpg"
 
+vision:
+  humanDetection:
+    enabled: false
+    confidenceThreshold: 0.55
+    # Continuous presence windows:
+    # - 30s => rover TTS warning
+    # - 60s => Discord human alert
+    ttsDelayMs: 30000
+    discordDelayMs: 60000
+    # Tolerates brief detector misses before declaring "cleared".
+    clearWindowMs: 3000
+    # After a Discord alert, require clear + cooldown before next alert.
+    cooldownMs: 900000
+    maxInferenceFpsPerCamera: 3
+
 discord:
   token: "DISCORD_BOT_TOKEN"
   guildId: "123456789012345678" # optional; bot works in any guild it's invited to
@@ -61,9 +76,11 @@ discord:
     adminAlerts: "123456789012345678"
     # chat bridge is configured per guild via `rs bridge` commands
     replay: "123456789012345678"
+    humanAlerts: "123456789012345678"
   roles:
     announcementPing: "123456789012345678"
     adminPing: "123456789012345678"
+    humanAlertPing: "123456789012345678"
 
 socials:
   - id: "discord"

server/index.js

@@ -27,6 +27,7 @@ require('./src/services/videoAuthService');
 require('./src/services/videoSocketService');
 require('./src/services/roomCameraSocketService');
 require('./src/services/roverSnapshotSocketService');
+require('./src/services/roomHumanDetectionService');
 require('./src/services/embedHttpService');
 require('./src/services/logStreamService');
 require('./src/services/adminLogService');

server/install_server.sh

@@ -10,6 +10,9 @@ MEDIAMTX_SERVICE="/etc/systemd/system/mediamtx.service"
 MULTIROVER_SERVICE="/etc/systemd/system/multirover.service"
 SNAPSHOT_DIR="/var/lib/rover-snapshots"
 REPLAY_SEGMENT_DIR="/var/lib/replay-segments"
+VISION_HOME="/opt/multiroomba-vision"
+VISION_VENV="$VISION_HOME/.venv"
+VISION_PYTHON="$VISION_VENV/bin/python3"
 
 if [[ $EUID -ne 0 ]]; then
   echo "This installer must be run with sudo/root." >&2
@@ -27,11 +30,11 @@ SERVER_DIR="$SCRIPT_DIR"
 CONFIG_PATH="$SERVER_DIR/config.yaml"
 MEDIAMTX_TEMPLATE="$SERVER_DIR/mediamtx/mediamtx.yml"
 
-echo "[1/6] Installing dependencies..."
-dnf install -y nodejs npm curl tar >/dev/null
+echo "[1/7] Installing dependencies..."
+dnf install -y nodejs npm curl tar python3 python3-pip >/dev/null
 NODE_BIN="$(command -v node)"
 
-echo "[2/6] Installing Node production deps..."
+echo "[2/7] Installing Node production deps..."
 runuser -u "$TARGET_USER" -- bash -c "cd '$SERVER_DIR' && npm install --production"
 
 if [[ ! -f "$CONFIG_PATH" ]]; then
@@ -40,6 +43,18 @@ if [[ ! -f "$CONFIG_PATH" ]]; then
   echo "Copied config.example.yaml to config.yaml; edit it before exposing the service."
 fi
 
+echo "[3/7] Installing OpenCV runtime..."
+mkdir -p "$VISION_HOME"
+python3 -m venv "$VISION_VENV"
+"$VISION_PYTHON" -m pip install --upgrade pip >/dev/null
+"$VISION_PYTHON" -m pip install "opencv-python-headless==4.10.0.84" "numpy==1.26.4" >/dev/null
+"$VISION_PYTHON" - <<'PY'
+import cv2
+import numpy
+print("Vision runtime OK", cv2.__version__, numpy.__version__)
+PY
+chown -R "$TARGET_USER":"$TARGET_USER" "$VISION_HOME"
+
 tmpdir=$(mktemp -d)
 trap 'rm -rf "$tmpdir"' EXIT
 
@@ -60,7 +75,7 @@ case "$arch" in
     ;;
 esac
 
-echo "[3/6] Installing mediaMTX ${MEDIAMTX_VERSION}..."
+echo "[4/7] Installing mediaMTX ${MEDIAMTX_VERSION}..."
 curl -L "$MEDIAMTX_BASE_URL/$mediamtx_pkg" -o "$tmpdir/mediamtx.tgz"
 tar -xzf "$tmpdir/mediamtx.tgz" -C "$tmpdir" mediamtx
 install -m 0755 "$tmpdir/mediamtx" "$MEDIAMTX_BIN"
@@ -75,7 +90,7 @@ rm -f "$MEDIAMTX_CONFIG"
 install -m 0644 "$MEDIAMTX_TEMPLATE" "$MEDIAMTX_CONFIG"
 chown -R "$TARGET_USER":"$TARGET_USER" "$MEDIAMTX_CONF_DIR"
 
-echo "[4/6] Writing systemd units..."
+echo "[5/7] Writing systemd units..."
 mkdir -p "$SNAPSHOT_DIR"
 chown "$TARGET_USER":"$TARGET_USER" "$SNAPSHOT_DIR"
 mkdir -p "$REPLAY_SEGMENT_DIR"
@@ -113,6 +128,7 @@ Environment=NODE_ENV=production
 Environment=SERVER_CONFIG=$CONFIG_PATH
 Environment=ROVER_SNAPSHOT_DIR=$SNAPSHOT_DIR
 Environment=REPLAY_SEGMENT_DIR=$REPLAY_SEGMENT_DIR
+Environment=VISION_PYTHON=$VISION_PYTHON
 ExecStart=$NODE_BIN $SERVER_DIR/index.js
 Restart=on-failure
 RestartSec=2
@@ -123,14 +139,14 @@ EOF
 
 chmod 644 "$MEDIAMTX_SERVICE" "$MULTIROVER_SERVICE"
 
-echo "[5/6] Enabling services..."
+echo "[6/7] Enabling services..."
 systemctl daemon-reload
 systemctl enable --now mediamtx.service
 systemctl enable --now multirover.service
 systemctl restart mediamtx.service
 systemctl restart multirover.service
 
-echo "[6/6] Done."
+echo "[7/7] Done."
 echo
 echo "Services installed:"
 echo "  mediamtx.service (WebRTC fan-out)"

server/scripts/human_detector_worker.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+import base64
+import json
+import sys
+from datetime import datetime, timezone
+
+import cv2
+import numpy as np
+
+
+MAX_EDGE = 640
+
+
+def now_iso():
+    return datetime.now(timezone.utc).isoformat()
+
+
+def emit(payload):
+    sys.stdout.write(json.dumps(payload, separators=(",", ":")) + "\n")
+    sys.stdout.flush()
+
+
+def decode_image(image_b64):
+    raw = base64.b64decode(image_b64)
+    arr = np.frombuffer(raw, dtype=np.uint8)
+    frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
+    if frame is None:
+        raise ValueError("failed to decode image")
+    return frame
+
+
+def resize_for_inference(frame):
+    h, w = frame.shape[:2]
+    max_side = max(h, w)
+    if max_side <= MAX_EDGE:
+        return frame
+    scale = float(MAX_EDGE) / float(max_side)
+    nw = max(1, int(round(w * scale)))
+    nh = max(1, int(round(h * scale)))
+    return cv2.resize(frame, (nw, nh), interpolation=cv2.INTER_AREA)
+
+
+def encode_jpeg(frame):
+    ok, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 78])
+    if not ok:
+        raise ValueError("failed to encode image")
+    return base64.b64encode(encoded.tobytes()).decode("ascii")
+
+
+def annotate_frame(frame, camera_id, confidence):
+    stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    msg = f"person {confidence:.2f} | {camera_id} | {stamp}"
+    cv2.rectangle(frame, (8, 8), (min(frame.shape[1] - 8, 540), 42), (0, 0, 0), -1)
+    cv2.putText(
+        frame,
+        msg,
+        (14, 30),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (0, 255, 0),
+        2,
+        cv2.LINE_AA,
+    )
+    return frame
+
+
+hog = cv2.HOGDescriptor()
+hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
+
+emit({"type": "ready", "ts": now_iso()})
+
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+    req = None
+    try:
+        req = json.loads(line)
+        req_id = req.get("reqId")
+        cam_id = str(req.get("cameraId", "unknown"))
+        conf_threshold = float(req.get("confidenceThreshold", 0.55))
+        frame = decode_image(req["imageBase64"])
+        working = resize_for_inference(frame)
+        rects, weights = hog.detectMultiScale(
+            working,
+            winStride=(8, 8),
+            padding=(8, 8),
+            scale=1.05,
+        )
+        best = 0.0
+        detections = []
+        if weights is None:
+            weights = []
+        for i, rect in enumerate(rects):
+            weight = float(weights[i]) if i < len(weights) else 0.0
+            best = max(best, weight)
+            if weight < conf_threshold:
+                continue
+            x, y, w, h = rect
+            detections.append({"x": int(x), "y": int(y), "w": int(w), "h": int(h), "confidence": weight})
+            cv2.rectangle(working, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 0), 2)
+            cv2.putText(
+                working,
+                f"{weight:.2f}",
+                (int(x), max(12, int(y) - 6)),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                0.5,
+                (0, 255, 0),
+                1,
+                cv2.LINE_AA,
+            )
+        detected = len(detections) > 0
+        if detected:
+            working = annotate_frame(working, cam_id, max(d["confidence"] for d in detections))
+        emit(
+            {
+                "ok": True,
+                "reqId": req_id,
+                "cameraId": cam_id,
+                "personDetected": detected,
+                "bestConfidence": best,
+                "detections": detections,
+                "annotatedBase64": encode_jpeg(working),
+                "ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000),
+            }
+        )
+    except Exception as err:
+        emit(
+            {
+                "ok": False,
+                "reqId": req.get("reqId") if isinstance(req, dict) else None,
+                "error": str(err),
+                "ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000),
+            }
+        )

server/src/services/discordBotService.js

@@ -1231,6 +1231,7 @@ async function announce({
   title,
   description,
   embeds,
+  files,
 }) {
   if (!channelId) return;
   const mentionChunks = [];
@@ -1247,7 +1248,7 @@ async function announce({
   await sendToChannel(
     channelId,
     `${prefix}${content || ''}`.trim(),
-    { embeds: payloadEmbeds },
+    { embeds: payloadEmbeds, files: Array.isArray(files) ? files : undefined },
     allowedMentions,
     !pingRoleId, // keep mention intact when pinging
   );
@@ -1459,6 +1460,43 @@ function handleBusEvent(event) {
         logger.warn('Replay send failed', err.message);
       });
       break;
+    case 'vision.humanDetected': {
+      const imageBase64 = payload?.imageBase64 ? String(payload.imageBase64) : '';
+      let attachment = null;
+      if (imageBase64) {
+        try {
+          const imageBuffer = Buffer.from(imageBase64, 'base64');
+          if (imageBuffer.length > 0) {
+            attachment = new AttachmentBuilder(imageBuffer, { name: 'human-detected.jpg' });
+          }
+        } catch (err) {
+          logger.warn('Failed to decode human detection image for Discord', err.message);
+        }
+      }
+      const cameraLabel = payload?.cameraId ? `Camera: \`${payload.cameraId}\`` : 'Camera: unknown';
+      const confidenceLabel =
+        Number.isFinite(Number(payload?.confidence))
+          ? `Confidence: \`${Number(payload.confidence).toFixed(2)}\``
+          : 'Confidence: n/a';
+      const detectedAt = Number(payload?.detectedAt);
+      const detectedLabel = Number.isFinite(detectedAt)
+        ? `Detected: <t:${Math.floor(detectedAt / 1000)}:F>`
+        : null;
+      const embed = buildEmbed({
+        title: 'Human Detected',
+        description: [cameraLabel, confidenceLabel, detectedLabel].filter(Boolean).join('\n'),
+        color: 0xe53935,
+      });
+      announce({
+        channelId: channels.humanAlerts,
+        pingRoleId: roles.humanAlertPing || null,
+        content: payload?.message || 'Human detected in room.',
+        embeds: [embed],
+        files: attachment ? [attachment] : [],
+        includeSiteUrl: false,
+      });
+      break;
+    }
     default:
       break;
   }

server/src/services/roomHumanDetectionService.js

@@ -0,0 +1,291 @@
+const path = require('path');
+const { spawn } = require('child_process');
+const readline = require('readline');
+const logger = require('../globals/logger').child('roomHumanDetection');
+const { loadConfig } = require('../helpers/configLoader');
+const { roomCameraStreamEvents } = require('./roomCameraSnapshotService');
+const roverManager = require('./roverManager');
+const { issueCommand } = require('./commandService');
+const { publishEvent } = require('./eventBus');
+const { sendAlert } = require('./alertService');
+
+const config = loadConfig();
+const visionConfig = config.vision?.humanDetection || {};
+const enabled = Boolean(visionConfig.enabled);
+
+if (!enabled) {
+  logger.info('Room human detection disabled via config');
+  return;
+}
+
+const CONFIDENCE_THRESHOLD = Number.isFinite(Number(visionConfig.confidenceThreshold))
+  ? Number(visionConfig.confidenceThreshold)
+  : 0.55;
+const TTS_DELAY_MS = Number.isFinite(Number(visionConfig.ttsDelayMs))
+  ? Number(visionConfig.ttsDelayMs)
+  : 30000;
+const DISCORD_DELAY_MS = Number.isFinite(Number(visionConfig.discordDelayMs))
+  ? Number(visionConfig.discordDelayMs)
+  : 60000;
+const CLEAR_WINDOW_MS = Number.isFinite(Number(visionConfig.clearWindowMs))
+  ? Number(visionConfig.clearWindowMs)
+  : 3000;
+const COOLDOWN_MS = Number.isFinite(Number(visionConfig.cooldownMs))
+  ? Number(visionConfig.cooldownMs)
+  : 15 * 60 * 1000;
+const MAX_INFERENCE_FPS = Number.isFinite(Number(visionConfig.maxInferenceFpsPerCamera))
+  ? Math.max(0.25, Number(visionConfig.maxInferenceFpsPerCamera))
+  : 3;
+const TTS_TEXT = 'human detected in room, alerting soon';
+
+const workerScript = path.join(__dirname, '..', '..', 'scripts', 'human_detector_worker.py');
+const pythonCandidates = [process.env.VISION_PYTHON, '/opt/multiroomba-vision/.venv/bin/python3', 'python3'].filter(Boolean);
+const inferenceIntervalMs = Math.max(100, Math.round(1000 / MAX_INFERENCE_FPS));
+
+const cameraState = new Map(); // cameraId -> { inflight,lastInferAt,lastResultAt,lastPositiveAt,lastConfidence,error }
+let worker = null;
+let workerReady = false;
+let reqSeq = 1;
+
+let lastAnyPositiveAt = null;
+let episodeStartAt = null;
+let ttsSentThisEpisode = false;
+let discordSentThisEpisode = false;
+let latestPositiveFrame = null; // { cameraId, confidence, ts, buffer }
+let cooldownUntil = 0;
+let hasClearedSinceLastDiscord = true;
+
+function ensureCameraState(cameraId) {
+  if (!cameraState.has(cameraId)) {
+    cameraState.set(cameraId, {
+      inflight: false,
+      lastInferAt: 0,
+      lastResultAt: 0,
+      lastPositiveAt: 0,
+      lastConfidence: 0,
+      error: null,
+    });
+  }
+  return cameraState.get(cameraId);
+}
+
+function markCleared(now) {
+  if (episodeStartAt == null) return;
+  episodeStartAt = null;
+  ttsSentThisEpisode = false;
+  discordSentThisEpisode = false;
+  lastAnyPositiveAt = null;
+  latestPositiveFrame = null;
+  hasClearedSinceLastDiscord = true;
+  logger.info('Human detection episode cleared', { now });
+}
+
+function sendTtsToNonPrivateRovers() {
+  const roster = roverManager.getRoster();
+  let sent = 0;
+  roster.forEach((entry) => {
+    if (entry?.private?.enabled) return;
+    try {
+      issueCommand(String(entry.id), {
+        type: 'tts',
+        tts: {
+          text: TTS_TEXT,
+          speak: true,
+        },
+      });
+      sent += 1;
+    } catch (err) {
+      logger.warn('Failed to send human-alert TTS', { roverId: entry?.id, error: err.message });
+    }
+  });
+  sendAlert({
+    color: '#f0b651',
+    title: 'Human Detection',
+    message: sent > 0 ? `Person detected; announced on ${sent} rover(s).` : 'Person detected; no non-private rover available.',
+  });
+  publishEvent({
+    source: 'roomHumanDetection',
+    type: 'vision.humanTtsSent',
+    payload: {
+      text: TTS_TEXT,
+      roverCount: sent,
+      ts: Date.now(),
+    },
+  });
+}
+
+function sendDiscordDetectionAlert(now) {
+  const payload = {
+    message: 'Human detected in room cameras.',
+    detectedAt: now,
+    confidence: latestPositiveFrame?.confidence || null,
+    cameraId: latestPositiveFrame?.cameraId || null,
+    imageBase64: latestPositiveFrame?.buffer ? latestPositiveFrame.buffer.toString('base64') : null,
+  };
+  publishEvent({
+    source: 'roomHumanDetection',
+    type: 'vision.humanDetected',
+    payload,
+  });
+  sendAlert({
+    color: '#e53935',
+    title: 'Human Detection',
+    message: 'Human presence persisted; Discord alert sent.',
+  });
+}
+
+function evaluateEpisode(now = Date.now()) {
+  if (episodeStartAt != null && lastAnyPositiveAt != null && now - lastAnyPositiveAt > CLEAR_WINDOW_MS) {
+    markCleared(now);
+    return;
+  }
+  if (episodeStartAt == null) return;
+  const elapsed = Math.max(0, now - episodeStartAt);
+  if (!ttsSentThisEpisode && elapsed >= TTS_DELAY_MS) {
+    sendTtsToNonPrivateRovers();
+    ttsSentThisEpisode = true;
+  }
+  if (discordSentThisEpisode) return;
+  if (elapsed < DISCORD_DELAY_MS) return;
+  if (!hasClearedSinceLastDiscord) return;
+  if (now < cooldownUntil) return;
+  sendDiscordDetectionAlert(now);
+  discordSentThisEpisode = true;
+  hasClearedSinceLastDiscord = false;
+  cooldownUntil = now + COOLDOWN_MS;
+}
+
+function handleWorkerMessage(line) {
+  let msg;
+  try {
+    msg = JSON.parse(line);
+  } catch (err) {
+    logger.warn('Invalid vision worker JSON', { error: err.message });
+    return;
+  }
+  if (msg?.type === 'ready') {
+    workerReady = true;
+    logger.info('Vision worker ready');
+    return;
+  }
+  const cameraId = String(msg?.cameraId || '');
+  if (!cameraId) return;
+  const state = ensureCameraState(cameraId);
+  state.inflight = false;
+  state.lastResultAt = Date.now();
+  if (!msg.ok) {
+    state.error = msg.error || 'worker error';
+    logger.warn('Vision inference failed', { cameraId, error: state.error });
+    evaluateEpisode(Date.now());
+    return;
+  }
+  state.error = null;
+  state.lastConfidence = Number(msg.bestConfidence || 0);
+  if (msg.personDetected) {
+    const now = Number(msg.ts) || Date.now();
+    state.lastPositiveAt = now;
+    lastAnyPositiveAt = now;
+    if (episodeStartAt == null) {
+      episodeStartAt = now;
+      ttsSentThisEpisode = false;
+      discordSentThisEpisode = false;
+      logger.info('Human detection episode started', { cameraId, now });
+    }
+    let buffer = null;
+    try {
+      if (msg.annotatedBase64) {
+        buffer = Buffer.from(String(msg.annotatedBase64), 'base64');
+      }
+    } catch (err) {
+      logger.warn('Failed to decode annotated frame from worker', { cameraId, error: err.message });
+    }
+    if (buffer) {
+      latestPositiveFrame = {
+        cameraId,
+        confidence: state.lastConfidence,
+        ts: now,
+        buffer,
+      };
+    }
+  }
+  evaluateEpisode(Number(msg.ts) || Date.now());
+}
+
+function startWorker() {
+  for (const pythonBin of pythonCandidates) {
+    try {
+      const proc = spawn(pythonBin, [workerScript], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+      worker = proc;
+      workerReady = false;
+      readline
+        .createInterface({ input: proc.stdout })
+        .on('line', handleWorkerMessage);
+      proc.stderr.on('data', (chunk) => {
+        const text = String(chunk || '').trim();
+        if (!text) return;
+        logger.warn('Vision worker stderr', { text: text.slice(0, 300) });
+      });
+      proc.on('exit', (code, signal) => {
+        logger.warn('Vision worker exited', { code, signal });
+        worker = null;
+        workerReady = false;
+      });
+      logger.info('Vision worker started', { pythonBin, workerScript });
+      return true;
+    } catch (err) {
+      logger.warn('Failed to spawn vision worker candidate', { pythonBin, error: err.message });
+    }
+  }
+  return false;
+}
+
+function submitFrame(cameraId, buffer, ts = Date.now()) {
+  if (!worker || !workerReady) return;
+  const state = ensureCameraState(cameraId);
+  const now = Date.now();
+  if (state.inflight) return;
+  if (now - state.lastInferAt < inferenceIntervalMs) return;
+  state.inflight = true;
+  state.lastInferAt = now;
+  const payload = {
+    reqId: `r${reqSeq++}`,
+    cameraId: String(cameraId),
+    ts: Number(ts) || now,
+    confidenceThreshold: CONFIDENCE_THRESHOLD,
+    imageBase64: buffer.toString('base64'),
+  };
+  try {
+    worker.stdin.write(`${JSON.stringify(payload)}\n`);
+  } catch (err) {
+    state.inflight = false;
+    state.error = err.message;
+    logger.warn('Failed writing frame to vision worker', { cameraId, error: err.message });
+  }
+}
+
+if (!startWorker()) {
+  logger.error('Room human detection disabled; failed to start Python worker');
+  return;
+}
+
+roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => {
+  if (!id || !buffer) return;
+  submitFrame(String(id), buffer, ts);
+});
+
+setInterval(() => {
+  evaluateEpisode(Date.now());
+}, 1000);
+
+logger.info('Room human detection service started', {
+  confidenceThreshold: CONFIDENCE_THRESHOLD,
+  ttsDelayMs: TTS_DELAY_MS,
+  discordDelayMs: DISCORD_DELAY_MS,
+  clearWindowMs: CLEAR_WINDOW_MS,
+  cooldownMs: COOLDOWN_MS,
+  maxInferenceFpsPerCamera: MAX_INFERENCE_FPS,
+});
+
+module.exports = {};