opencv

plans/to-do.md

@@ -2,7 +2,7 @@
 2. trusted user system [x]
 3. private rovers [x]
 4. optional bump-off in drive macro [x]
-5. allow admins to click on locked rovers from the roster
+5. allow admins to click on locked rovers from the roster [x]
 6. add faster way for admins to login
 7. custom webhook profile pictures for chat bridge in discord
 8. home assistant switch that tells the server to force the lights on

server/config.example.yaml

@@ -52,6 +52,21 @@ roomCameras:
     url: "http://192.168.0.51/snapshot.jpg"
     streamUrl: "http://192.168.0.51/stream.mjpg"
 
+vision:
+  humanDetection:
+    enabled: false
+    confidenceThreshold: 0.55
+    # Continuous presence windows:
+    # - 30s => rover TTS warning
+    # - 60s => Discord human alert
+    ttsDelayMs: 30000
+    discordDelayMs: 60000
+    # Tolerates brief detector misses before declaring "cleared".
+    clearWindowMs: 3000
+    # After a Discord alert, require clear + cooldown before next alert.
+    cooldownMs: 900000
+    maxInferenceFpsPerCamera: 3
+
 discord:
   token: "DISCORD_BOT_TOKEN"
   guildId: "123456789012345678" # optional; bot works in any guild it's invited to
@@ -61,9 +76,11 @@ discord:
     adminAlerts: "123456789012345678"
     # chat bridge is configured per guild via `rs bridge` commands
     replay: "123456789012345678"
+    humanAlerts: "123456789012345678"
   roles:
     announcementPing: "123456789012345678"
     adminPing: "123456789012345678"
+    humanAlertPing: "123456789012345678"
 
 socials:
   - id: "discord"

server/index.js

@@ -27,6 +27,7 @@ require('./src/services/videoAuthService');
 require('./src/services/videoSocketService');
 require('./src/services/roomCameraSocketService');
 require('./src/services/roverSnapshotSocketService');
+require('./src/services/roomHumanDetectionService');
 require('./src/services/embedHttpService');
 require('./src/services/logStreamService');
 require('./src/services/adminLogService');

server/install_server.sh

@@ -10,6 +10,9 @@ MEDIAMTX_SERVICE="/etc/systemd/system/mediamtx.service"
 MULTIROVER_SERVICE="/etc/systemd/system/multirover.service"
 SNAPSHOT_DIR="/var/lib/rover-snapshots"
 REPLAY_SEGMENT_DIR="/var/lib/replay-segments"
+VISION_HOME="/opt/multiroomba-vision"
+VISION_VENV="$VISION_HOME/.venv"
+VISION_PYTHON="$VISION_VENV/bin/python3"
 
 if [[ $EUID -ne 0 ]]; then
   echo "This installer must be run with sudo/root." >&2
@@ -27,11 +30,11 @@ SERVER_DIR="$SCRIPT_DIR"
 CONFIG_PATH="$SERVER_DIR/config.yaml"
 MEDIAMTX_TEMPLATE="$SERVER_DIR/mediamtx/mediamtx.yml"
 
-echo "[1/6] Installing dependencies..."
+echo "[1/7] Installing dependencies..."
-dnf install -y nodejs npm curl tar >/dev/null
+dnf install -y nodejs npm curl tar python3 python3-pip >/dev/null
 NODE_BIN="$(command -v node)"
 
-echo "[2/6] Installing Node production deps..."
+echo "[2/7] Installing Node production deps..."
 runuser -u "$TARGET_USER" -- bash -c "cd '$SERVER_DIR' && npm install --production"
 
 if [[ ! -f "$CONFIG_PATH" ]]; then
@@ -40,6 +43,18 @@ if [[ ! -f "$CONFIG_PATH" ]]; then
   echo "Copied config.example.yaml to config.yaml; edit it before exposing the service."
 fi
 
+echo "[3/7] Installing OpenCV runtime..."
+mkdir -p "$VISION_HOME"
+python3 -m venv "$VISION_VENV"
+"$VISION_PYTHON" -m pip install --upgrade pip >/dev/null
+"$VISION_PYTHON" -m pip install "opencv-python-headless==4.10.0.84" "numpy==1.26.4" >/dev/null
+"$VISION_PYTHON" - <<'PY'
+import cv2
+import numpy
+print("Vision runtime OK", cv2.__version__, numpy.__version__)
+PY
+chown -R "$TARGET_USER":"$TARGET_USER" "$VISION_HOME"
+
 tmpdir=$(mktemp -d)
 trap 'rm -rf "$tmpdir"' EXIT
 
@@ -60,7 +75,7 @@ case "$arch" in
     ;;
 esac
 
-echo "[3/6] Installing mediaMTX ${MEDIAMTX_VERSION}..."
+echo "[4/7] Installing mediaMTX ${MEDIAMTX_VERSION}..."
 curl -L "$MEDIAMTX_BASE_URL/$mediamtx_pkg" -o "$tmpdir/mediamtx.tgz"
 tar -xzf "$tmpdir/mediamtx.tgz" -C "$tmpdir" mediamtx
 install -m 0755 "$tmpdir/mediamtx" "$MEDIAMTX_BIN"
@@ -75,7 +90,7 @@ rm -f "$MEDIAMTX_CONFIG"
 install -m 0644 "$MEDIAMTX_TEMPLATE" "$MEDIAMTX_CONFIG"
 chown -R "$TARGET_USER":"$TARGET_USER" "$MEDIAMTX_CONF_DIR"
 
-echo "[4/6] Writing systemd units..."
+echo "[5/7] Writing systemd units..."
 mkdir -p "$SNAPSHOT_DIR"
 chown "$TARGET_USER":"$TARGET_USER" "$SNAPSHOT_DIR"
 mkdir -p "$REPLAY_SEGMENT_DIR"
@@ -113,6 +128,7 @@ Environment=NODE_ENV=production
 Environment=SERVER_CONFIG=$CONFIG_PATH
 Environment=ROVER_SNAPSHOT_DIR=$SNAPSHOT_DIR
 Environment=REPLAY_SEGMENT_DIR=$REPLAY_SEGMENT_DIR
+Environment=VISION_PYTHON=$VISION_PYTHON
 ExecStart=$NODE_BIN $SERVER_DIR/index.js
 Restart=on-failure
 RestartSec=2
@@ -123,14 +139,14 @@ EOF
 
 chmod 644 "$MEDIAMTX_SERVICE" "$MULTIROVER_SERVICE"
 
-echo "[5/6] Enabling services..."
+echo "[6/7] Enabling services..."
 systemctl daemon-reload
 systemctl enable --now mediamtx.service
 systemctl enable --now multirover.service
 systemctl restart mediamtx.service
 systemctl restart multirover.service
 
-echo "[6/6] Done."
+echo "[7/7] Done."
 echo
 echo "Services installed:"
 echo "  mediamtx.service (WebRTC fan-out)"

server/scripts/human_detector_worker.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+import base64
+import json
+import sys
+from datetime import datetime, timezone
+
+import cv2
+import numpy as np
+
+
+MAX_EDGE = 640
+
+
+def now_iso():
+    return datetime.now(timezone.utc).isoformat()
+
+
+def emit(payload):
+    sys.stdout.write(json.dumps(payload, separators=(",", ":")) + "\n")
+    sys.stdout.flush()
+
+
+def decode_image(image_b64):
+    raw = base64.b64decode(image_b64)
+    arr = np.frombuffer(raw, dtype=np.uint8)
+    frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
+    if frame is None:
+        raise ValueError("failed to decode image")
+    return frame
+
+
+def resize_for_inference(frame):
+    h, w = frame.shape[:2]
+    max_side = max(h, w)
+    if max_side <= MAX_EDGE:
+        return frame
+    scale = float(MAX_EDGE) / float(max_side)
+    nw = max(1, int(round(w * scale)))
+    nh = max(1, int(round(h * scale)))
+    return cv2.resize(frame, (nw, nh), interpolation=cv2.INTER_AREA)
+
+
+def encode_jpeg(frame):
+    ok, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 78])
+    if not ok:
+        raise ValueError("failed to encode image")
+    return base64.b64encode(encoded.tobytes()).decode("ascii")
+
+
+def annotate_frame(frame, camera_id, confidence):
+    stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    msg = f"person {confidence:.2f} | {camera_id} | {stamp}"
+    cv2.rectangle(frame, (8, 8), (min(frame.shape[1] - 8, 540), 42), (0, 0, 0), -1)
+    cv2.putText(
+        frame,
+        msg,
+        (14, 30),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (0, 255, 0),
+        2,
+        cv2.LINE_AA,
+    )
+    return frame
+
+
+hog = cv2.HOGDescriptor()
+hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
+
+emit({"type": "ready", "ts": now_iso()})
+
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+    req = None
+    try:
+        req = json.loads(line)
+        req_id = req.get("reqId")
+        cam_id = str(req.get("cameraId", "unknown"))
+        conf_threshold = float(req.get("confidenceThreshold", 0.55))
+        frame = decode_image(req["imageBase64"])
+        working = resize_for_inference(frame)
+        rects, weights = hog.detectMultiScale(
+            working,
+            winStride=(8, 8),
+            padding=(8, 8),
+            scale=1.05,
+        )
+        best = 0.0
+        detections = []
+        if weights is None:
+            weights = []
+        for i, rect in enumerate(rects):
+            weight = float(weights[i]) if i < len(weights) else 0.0
+            best = max(best, weight)
+            if weight < conf_threshold:
+                continue
+            x, y, w, h = rect
+            detections.append({"x": int(x), "y": int(y), "w": int(w), "h": int(h), "confidence": weight})
+            cv2.rectangle(working, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 0), 2)
+            cv2.putText(
+                working,
+                f"{weight:.2f}",
+                (int(x), max(12, int(y) - 6)),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                0.5,
+                (0, 255, 0),
+                1,
+                cv2.LINE_AA,
+            )
+        detected = len(detections) > 0
+        if detected:
+            working = annotate_frame(working, cam_id, max(d["confidence"] for d in detections))
+        emit(
+            {
+                "ok": True,
+                "reqId": req_id,
+                "cameraId": cam_id,
+                "personDetected": detected,
+                "bestConfidence": best,
+                "detections": detections,
+                "annotatedBase64": encode_jpeg(working),
+                "ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000),
+            }
+        )
+    except Exception as err:
+        emit(
+            {
+                "ok": False,
+                "reqId": req.get("reqId") if isinstance(req, dict) else None,
+                "error": str(err),
+                "ts": int(datetime.now(tz=timezone.utc).timestamp() * 1000),
+            }
+        )

server/src/services/discordBotService.js

@@ -1231,6 +1231,7 @@ async function announce({
   title,
   description,
   embeds,
+  files,
 }) {
   if (!channelId) return;
   const mentionChunks = [];
@@ -1247,7 +1248,7 @@ async function announce({
   await sendToChannel(
     channelId,
     `${prefix}${content || ''}`.trim(),
-    { embeds: payloadEmbeds },
+    { embeds: payloadEmbeds, files: Array.isArray(files) ? files : undefined },
     allowedMentions,
     !pingRoleId, // keep mention intact when pinging
   );
@@ -1459,6 +1460,43 @@ function handleBusEvent(event) {
         logger.warn('Replay send failed', err.message);
       });
       break;
+    case 'vision.humanDetected': {
+      const imageBase64 = payload?.imageBase64 ? String(payload.imageBase64) : '';
+      let attachment = null;
+      if (imageBase64) {
+        try {
+          const imageBuffer = Buffer.from(imageBase64, 'base64');
+          if (imageBuffer.length > 0) {
+            attachment = new AttachmentBuilder(imageBuffer, { name: 'human-detected.jpg' });
+          }
+        } catch (err) {
+          logger.warn('Failed to decode human detection image for Discord', err.message);
+        }
+      }
+      const cameraLabel = payload?.cameraId ? `Camera: \`${payload.cameraId}\`` : 'Camera: unknown';
+      const confidenceLabel =
+        Number.isFinite(Number(payload?.confidence))
+          ? `Confidence: \`${Number(payload.confidence).toFixed(2)}\``
+          : 'Confidence: n/a';
+      const detectedAt = Number(payload?.detectedAt);
+      const detectedLabel = Number.isFinite(detectedAt)
+        ? `Detected: <t:${Math.floor(detectedAt / 1000)}:F>`
+        : null;
+      const embed = buildEmbed({
+        title: 'Human Detected',
+        description: [cameraLabel, confidenceLabel, detectedLabel].filter(Boolean).join('\n'),
+        color: 0xe53935,
+      });
+      announce({
+        channelId: channels.humanAlerts,
+        pingRoleId: roles.humanAlertPing || null,
+        content: payload?.message || 'Human detected in room.',
+        embeds: [embed],
+        files: attachment ? [attachment] : [],
+        includeSiteUrl: false,
+      });
+      break;
+    }
     default:
       break;
   }

server/src/services/roomHumanDetectionService.js

@@ -0,0 +1,291 @@
+const path = require('path');
+const { spawn } = require('child_process');
+const readline = require('readline');
+const logger = require('../globals/logger').child('roomHumanDetection');
+const { loadConfig } = require('../helpers/configLoader');
+const { roomCameraStreamEvents } = require('./roomCameraSnapshotService');
+const roverManager = require('./roverManager');
+const { issueCommand } = require('./commandService');
+const { publishEvent } = require('./eventBus');
+const { sendAlert } = require('./alertService');
+
+const config = loadConfig();
+const visionConfig = config.vision?.humanDetection || {};
+const enabled = Boolean(visionConfig.enabled);
+
+if (!enabled) {
+  logger.info('Room human detection disabled via config');
+  return;
+}
+
+const CONFIDENCE_THRESHOLD = Number.isFinite(Number(visionConfig.confidenceThreshold))
+  ? Number(visionConfig.confidenceThreshold)
+  : 0.55;
+const TTS_DELAY_MS = Number.isFinite(Number(visionConfig.ttsDelayMs))
+  ? Number(visionConfig.ttsDelayMs)
+  : 30000;
+const DISCORD_DELAY_MS = Number.isFinite(Number(visionConfig.discordDelayMs))
+  ? Number(visionConfig.discordDelayMs)
+  : 60000;
+const CLEAR_WINDOW_MS = Number.isFinite(Number(visionConfig.clearWindowMs))
+  ? Number(visionConfig.clearWindowMs)
+  : 3000;
+const COOLDOWN_MS = Number.isFinite(Number(visionConfig.cooldownMs))
+  ? Number(visionConfig.cooldownMs)
+  : 15 * 60 * 1000;
+const MAX_INFERENCE_FPS = Number.isFinite(Number(visionConfig.maxInferenceFpsPerCamera))
+  ? Math.max(0.25, Number(visionConfig.maxInferenceFpsPerCamera))
+  : 3;
+const TTS_TEXT = 'human detected in room, alerting soon';
+
+const workerScript = path.join(__dirname, '..', '..', 'scripts', 'human_detector_worker.py');
+const pythonCandidates = [process.env.VISION_PYTHON, '/opt/multiroomba-vision/.venv/bin/python3', 'python3'].filter(Boolean);
+const inferenceIntervalMs = Math.max(100, Math.round(1000 / MAX_INFERENCE_FPS));
+
+const cameraState = new Map(); // cameraId -> { inflight,lastInferAt,lastResultAt,lastPositiveAt,lastConfidence,error }
+let worker = null;
+let workerReady = false;
+let reqSeq = 1;
+
+let lastAnyPositiveAt = null;
+let episodeStartAt = null;
+let ttsSentThisEpisode = false;
+let discordSentThisEpisode = false;
+let latestPositiveFrame = null; // { cameraId, confidence, ts, buffer }
+let cooldownUntil = 0;
+let hasClearedSinceLastDiscord = true;
+
+function ensureCameraState(cameraId) {
+  if (!cameraState.has(cameraId)) {
+    cameraState.set(cameraId, {
+      inflight: false,
+      lastInferAt: 0,
+      lastResultAt: 0,
+      lastPositiveAt: 0,
+      lastConfidence: 0,
+      error: null,
+    });
+  }
+  return cameraState.get(cameraId);
+}
+
+function markCleared(now) {
+  if (episodeStartAt == null) return;
+  episodeStartAt = null;
+  ttsSentThisEpisode = false;
+  discordSentThisEpisode = false;
+  lastAnyPositiveAt = null;
+  latestPositiveFrame = null;
+  hasClearedSinceLastDiscord = true;
+  logger.info('Human detection episode cleared', { now });
+}
+
+function sendTtsToNonPrivateRovers() {
+  const roster = roverManager.getRoster();
+  let sent = 0;
+  roster.forEach((entry) => {
+    if (entry?.private?.enabled) return;
+    try {
+      issueCommand(String(entry.id), {
+        type: 'tts',
+        tts: {
+          text: TTS_TEXT,
+          speak: true,
+        },
+      });
+      sent += 1;
+    } catch (err) {
+      logger.warn('Failed to send human-alert TTS', { roverId: entry?.id, error: err.message });
+    }
+  });
+  sendAlert({
+    color: '#f0b651',
+    title: 'Human Detection',
+    message: sent > 0 ? `Person detected; announced on ${sent} rover(s).` : 'Person detected; no non-private rover available.',
+  });
+  publishEvent({
+    source: 'roomHumanDetection',
+    type: 'vision.humanTtsSent',
+    payload: {
+      text: TTS_TEXT,
+      roverCount: sent,
+      ts: Date.now(),
+    },
+  });
+}
+
+function sendDiscordDetectionAlert(now) {
+  const payload = {
+    message: 'Human detected in room cameras.',
+    detectedAt: now,
+    confidence: latestPositiveFrame?.confidence || null,
+    cameraId: latestPositiveFrame?.cameraId || null,
+    imageBase64: latestPositiveFrame?.buffer ? latestPositiveFrame.buffer.toString('base64') : null,
+  };
+  publishEvent({
+    source: 'roomHumanDetection',
+    type: 'vision.humanDetected',
+    payload,
+  });
+  sendAlert({
+    color: '#e53935',
+    title: 'Human Detection',
+    message: 'Human presence persisted; Discord alert sent.',
+  });
+}
+
+function evaluateEpisode(now = Date.now()) {
+  if (episodeStartAt != null && lastAnyPositiveAt != null && now - lastAnyPositiveAt > CLEAR_WINDOW_MS) {
+    markCleared(now);
+    return;
+  }
+  if (episodeStartAt == null) return;
+  const elapsed = Math.max(0, now - episodeStartAt);
+  if (!ttsSentThisEpisode && elapsed >= TTS_DELAY_MS) {
+    sendTtsToNonPrivateRovers();
+    ttsSentThisEpisode = true;
+  }
+  if (discordSentThisEpisode) return;
+  if (elapsed < DISCORD_DELAY_MS) return;
+  if (!hasClearedSinceLastDiscord) return;
+  if (now < cooldownUntil) return;
+  sendDiscordDetectionAlert(now);
+  discordSentThisEpisode = true;
+  hasClearedSinceLastDiscord = false;
+  cooldownUntil = now + COOLDOWN_MS;
+}
+
+function handleWorkerMessage(line) {
+  let msg;
+  try {
+    msg = JSON.parse(line);
+  } catch (err) {
+    logger.warn('Invalid vision worker JSON', { error: err.message });
+    return;
+  }
+  if (msg?.type === 'ready') {
+    workerReady = true;
+    logger.info('Vision worker ready');
+    return;
+  }
+  const cameraId = String(msg?.cameraId || '');
+  if (!cameraId) return;
+  const state = ensureCameraState(cameraId);
+  state.inflight = false;
+  state.lastResultAt = Date.now();
+  if (!msg.ok) {
+    state.error = msg.error || 'worker error';
+    logger.warn('Vision inference failed', { cameraId, error: state.error });
+    evaluateEpisode(Date.now());
+    return;
+  }
+  state.error = null;
+  state.lastConfidence = Number(msg.bestConfidence || 0);
+  if (msg.personDetected) {
+    const now = Number(msg.ts) || Date.now();
+    state.lastPositiveAt = now;
+    lastAnyPositiveAt = now;
+    if (episodeStartAt == null) {
+      episodeStartAt = now;
+      ttsSentThisEpisode = false;
+      discordSentThisEpisode = false;
+      logger.info('Human detection episode started', { cameraId, now });
+    }
+    let buffer = null;
+    try {
+      if (msg.annotatedBase64) {
+        buffer = Buffer.from(String(msg.annotatedBase64), 'base64');
+      }
+    } catch (err) {
+      logger.warn('Failed to decode annotated frame from worker', { cameraId, error: err.message });
+    }
+    if (buffer) {
+      latestPositiveFrame = {
+        cameraId,
+        confidence: state.lastConfidence,
+        ts: now,
+        buffer,
+      };
+    }
+  }
+  evaluateEpisode(Number(msg.ts) || Date.now());
+}
+
+function startWorker() {
+  for (const pythonBin of pythonCandidates) {
+    try {
+      const proc = spawn(pythonBin, [workerScript], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+      worker = proc;
+      workerReady = false;
+      readline
+        .createInterface({ input: proc.stdout })
+        .on('line', handleWorkerMessage);
+      proc.stderr.on('data', (chunk) => {
+        const text = String(chunk || '').trim();
+        if (!text) return;
+        logger.warn('Vision worker stderr', { text: text.slice(0, 300) });
+      });
+      proc.on('exit', (code, signal) => {
+        logger.warn('Vision worker exited', { code, signal });
+        worker = null;
+        workerReady = false;
+      });
+      logger.info('Vision worker started', { pythonBin, workerScript });
+      return true;
+    } catch (err) {
+      logger.warn('Failed to spawn vision worker candidate', { pythonBin, error: err.message });
+    }
+  }
+  return false;
+}
+
+function submitFrame(cameraId, buffer, ts = Date.now()) {
+  if (!worker || !workerReady) return;
+  const state = ensureCameraState(cameraId);
+  const now = Date.now();
+  if (state.inflight) return;
+  if (now - state.lastInferAt < inferenceIntervalMs) return;
+  state.inflight = true;
+  state.lastInferAt = now;
+  const payload = {
+    reqId: `r${reqSeq++}`,
+    cameraId: String(cameraId),
+    ts: Number(ts) || now,
+    confidenceThreshold: CONFIDENCE_THRESHOLD,
+    imageBase64: buffer.toString('base64'),
+  };
+  try {
+    worker.stdin.write(`${JSON.stringify(payload)}\n`);
+  } catch (err) {
+    state.inflight = false;
+    state.error = err.message;
+    logger.warn('Failed writing frame to vision worker', { cameraId, error: err.message });
+  }
+}
+
+if (!startWorker()) {
+  logger.error('Room human detection disabled; failed to start Python worker');
+  return;
+}
+
+roomCameraStreamEvents.on('frame', ({ id, buffer, ts }) => {
+  if (!id || !buffer) return;
+  submitFrame(String(id), buffer, ts);
+});
+
+setInterval(() => {
+  evaluateEpisode(Date.now());
+}, 1000);
+
+logger.info('Room human detection service started', {
+  confidenceThreshold: CONFIDENCE_THRESHOLD,
+  ttsDelayMs: TTS_DELAY_MS,
+  discordDelayMs: DISCORD_DELAY_MS,
+  clearWindowMs: CLEAR_WINDOW_MS,
+  cooldownMs: COOLDOWN_MS,
+  maxInferenceFpsPerCamera: MAX_INFERENCE_FPS,
+});
+
+module.exports = {};