kinect stuff v1

webui/package-lock.json

@@ -12,7 +12,8 @@
         "react-dom": "^19.2.0",
         "react-icons": "^5.5.0",
         "react-joystick-component": "^6.2.1",
-        "react-router-dom": "^7.9.6"
+        "react-router-dom": "^7.9.6",
+        "three": "^0.184.0"
       },
       "devDependencies": {
         "@eslint/js": "^9.39.1",
@@ -3960,6 +3961,12 @@
         "node": ">=0.8"
       }
     },
+    "node_modules/three": {
+      "version": "0.184.0",
+      "resolved": "https://registry.npmjs.org/three/-/three-0.184.0.tgz",
+      "integrity": "sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==",
+      "license": "MIT"
+    },
     "node_modules/tinyglobby": {
       "version": "0.2.15",
       "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",

webui/package.json

@@ -14,7 +14,8 @@
     "react-dom": "^19.2.0",
     "react-icons": "^5.5.0",
     "react-joystick-component": "^6.2.1",
-    "react-router-dom": "^7.9.6"
+    "react-router-dom": "^7.9.6",
+    "three": "^0.184.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.1",
@@ -27,8 +28,8 @@
     "eslint-plugin-react-refresh": "^0.4.24",
     "globals": "^16.5.0",
     "postcss": "^8.5.6",
+    "socket.io-client": "4.8.3",
     "tailwindcss": "^3.4.14",
-    "vite": "^7.2.2",
-    "socket.io-client": "4.8.3"
+    "vite": "^7.2.2"
   }
 }

webui/src/App.jsx

@@ -16,6 +16,7 @@ import {
   useControlSystem,
 } from './controls/index.js';
 import RoomCameraPanel from './components/RoomCameraPanel/index.jsx';
+import KinectPanel from './components/KinectPanel/index.jsx';
 import LogPanel from './components/LogPanel/index.jsx';
 import DriverVideo from './components/DriverVideo/index.jsx';
 import RightPaneTabs from './components/RightPaneTabs/index.jsx';
@@ -167,6 +168,7 @@ function MobileFeatureTabs({
               <HomeAssistantControls />
               <ButtonBoxPanel />
               <RoomCameraPanel panelId={roomPanelId} />
+              <KinectPanel />
             </div>
           </TabPanel>
           <TabPanel id="help">

webui/src/components/KinectPanel/PointCloudViewer.jsx

@@ -0,0 +1,173 @@
+// Kinect Point Cloud Viewer
+// Purpose: Renders a requested Kinect point-cloud frame as an interactive local Three.js scene.
+// Scope: Owns lazy-loading Three.js, converting binary point data into geometry, and pausing work off-screen.
+import { useCallback, useEffect, useRef } from 'react';
+
+export default function PointCloudViewer({ frame }) {
+  const hostRef = useRef(null);
+  const rendererRef = useRef(null);
+  const cameraRef = useRef(null);
+  const sceneRef = useRef(null);
+  const pointsRef = useRef(null);
+  const controlsRef = useRef(null);
+  const threeRef = useRef(null);
+  const visibleRef = useRef(false);
+  const frameRef = useRef(frame);
+
+  const renderOnce = useCallback(() => {
+    const renderer = rendererRef.current;
+    const scene = sceneRef.current;
+    const camera = cameraRef.current;
+    if (!renderer || !scene || !camera || !visibleRef.current) return;
+    renderer.render(scene, camera);
+  }, []);
+
+  const rebuildGeometry = useCallback(() => {
+    const scene = sceneRef.current;
+    const currentFrame = frameRef.current;
+    const THREE = threeRef.current;
+    if (!scene || !THREE || !currentFrame?.buffer || !visibleRef.current) return;
+
+    const pointCount = Number(currentFrame.meta?.pointCount) || 0;
+    const strideBytes = Number(currentFrame.meta?.strideBytes) || 16;
+    if (!pointCount || strideBytes < 16) return;
+
+    const view = new DataView(currentFrame.buffer);
+    const positions = new Float32Array(pointCount * 3);
+    const colors = new Float32Array(pointCount * 3);
+
+    // The server sends x/y/z as little-endian floats followed by rgba bytes.
+    // Building typed arrays only when the canvas is visible keeps expensive
+    // browser-side point conversion from happening while the card is off-screen.
+    for (let index = 0; index < pointCount; index += 1) {
+      const source = index * strideBytes;
+      const target = index * 3;
+      positions[target + 0] = view.getFloat32(source + 0, true);
+      positions[target + 1] = view.getFloat32(source + 4, true);
+      positions[target + 2] = -view.getFloat32(source + 8, true);
+      colors[target + 0] = view.getUint8(source + 12) / 255;
+      colors[target + 1] = view.getUint8(source + 13) / 255;
+      colors[target + 2] = view.getUint8(source + 14) / 255;
+    }
+
+    const geometry = new THREE.BufferGeometry();
+    geometry.setAttribute('position', new THREE.BufferAttribute(positions, 3));
+    geometry.setAttribute('color', new THREE.BufferAttribute(colors, 3));
+    geometry.computeBoundingSphere();
+
+    const material = new THREE.PointsMaterial({
+      size: 0.018,
+      vertexColors: true,
+      sizeAttenuation: true,
+    });
+    const points = new THREE.Points(geometry, material);
+
+    if (pointsRef.current) {
+      scene.remove(pointsRef.current);
+      pointsRef.current.geometry.dispose();
+      pointsRef.current.material.dispose();
+    }
+    pointsRef.current = points;
+    scene.add(points);
+    renderOnce();
+  }, [renderOnce]);
+
+  useEffect(() => {
+    frameRef.current = frame;
+    rebuildGeometry();
+  }, [frame, rebuildGeometry]);
+
+  useEffect(() => {
+    const host = hostRef.current;
+    if (!host) return undefined;
+
+    let cancelled = false;
+    let resizeObserver = null;
+    let intersectionObserver = null;
+
+    async function initRenderer() {
+      // Three.js is loaded only after a point-cloud frame exists and this view
+      // mounts.  The import resolves from locally built assets, so the viewer
+      // still works without internet access.
+      const [threeModule, controlsModule] = await Promise.all([
+        import('three'),
+        import('three/examples/jsm/controls/OrbitControls.js'),
+      ]);
+      if (cancelled) return;
+
+      const THREE = threeModule;
+      const { OrbitControls } = controlsModule;
+      threeRef.current = THREE;
+
+      const scene = new THREE.Scene();
+      scene.background = new THREE.Color(0x0a0a0a);
+      const camera = new THREE.PerspectiveCamera(55, 4 / 3, 0.01, 20);
+      camera.position.set(0, 0.15, 2.2);
+      const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: false });
+      renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 1.5));
+      renderer.setSize(host.clientWidth || 640, host.clientHeight || 480, false);
+      host.appendChild(renderer.domElement);
+
+      const controls = new OrbitControls(camera, renderer.domElement);
+      controls.enableDamping = false;
+      controls.target.set(0, 0, -1.4);
+      controls.addEventListener('change', renderOnce);
+
+      sceneRef.current = scene;
+      cameraRef.current = camera;
+      rendererRef.current = renderer;
+      controlsRef.current = controls;
+
+      resizeObserver = new ResizeObserver(([entry]) => {
+        const width = Math.max(1, entry.contentRect.width);
+        const height = Math.max(1, entry.contentRect.height);
+        camera.aspect = width / height;
+        camera.updateProjectionMatrix();
+        renderer.setSize(width, height, false);
+        renderOnce();
+      });
+      resizeObserver.observe(host);
+
+      intersectionObserver = new IntersectionObserver(
+        ([entry]) => {
+          const nextVisible = Boolean(entry?.isIntersecting);
+          visibleRef.current = nextVisible;
+          if (nextVisible) {
+            // When the card becomes visible again, rebuild from the latest
+            // cached frame so users see current data without rendering while
+            // hidden.
+            rebuildGeometry();
+            renderOnce();
+          }
+        },
+        { threshold: 0.01 },
+      );
+      intersectionObserver.observe(host);
+    }
+
+    initRenderer();
+
+    return () => {
+      cancelled = true;
+      intersectionObserver?.disconnect();
+      resizeObserver?.disconnect();
+      controlsRef.current?.removeEventListener('change', renderOnce);
+      controlsRef.current?.dispose();
+      if (pointsRef.current) {
+        sceneRef.current?.remove(pointsRef.current);
+        pointsRef.current.geometry.dispose();
+        pointsRef.current.material.dispose();
+        pointsRef.current = null;
+      }
+      rendererRef.current?.dispose();
+      rendererRef.current?.domElement?.remove();
+      sceneRef.current = null;
+      cameraRef.current = null;
+      rendererRef.current = null;
+      controlsRef.current = null;
+      threeRef.current = null;
+    };
+  }, [rebuildGeometry, renderOnce]);
+
+  return <div ref={hostRef} className="h-full w-full" />;
+}

webui/src/components/KinectPanel/index.jsx

@@ -0,0 +1,154 @@
+// Kinect Panel
+// Purpose: Displays request-only Kinect image and 3D snapshots shared by the server.
+// Scope: Owns browser socket events, cached frame display, request controls, and card layout.
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useSocket } from '../../context/SocketContext.jsx';
+import { useSessionSelector } from '../../context/SessionContext.jsx';
+import CardFrame from '../CardFrame/index.jsx';
+import PointCloudViewer from './PointCloudViewer.jsx';
+import {
+  buildStatusPill,
+  normalizeBinaryPayload,
+  normalizeKinectStatus,
+} from './utils.js';
+
+export default function KinectPanel() {
+  const socket = useSocket();
+  const status = useSessionSelector((state) => normalizeKinectStatus(state.session?.kinect));
+  const [activeView, setActiveView] = useState('3d');
+  const [pointCloudFrame, setPointCloudFrame] = useState(null);
+  const [colorUrl, setColorUrl] = useState(null);
+  const [requestError, setRequestError] = useState(null);
+  const [nowMs, setNowMs] = useState(() => Date.now());
+
+  useEffect(() => {
+    const timer = setInterval(() => setNowMs(Date.now()), 1000);
+    return () => clearInterval(timer);
+  }, []);
+
+  useEffect(() => {
+    if (!socket) return undefined;
+
+    const handlePointCloudFrame = (meta = {}, buffer) => {
+      const normalized = normalizeBinaryPayload(buffer);
+      if (!normalized) return;
+      setPointCloudFrame({ meta, buffer: normalized });
+      setActiveView('3d');
+      setRequestError(null);
+    };
+
+    const handleColorFrame = (...args) => {
+      const buffer = args[1];
+      const normalized = normalizeBinaryPayload(buffer);
+      if (!normalized) return;
+      const blob = new Blob([normalized], { type: 'image/jpeg' });
+      const nextUrl = URL.createObjectURL(blob);
+      setColorUrl((previous) => {
+        if (previous) URL.revokeObjectURL(previous);
+        return nextUrl;
+      });
+      setActiveView('image');
+      setRequestError(null);
+    };
+
+    socket.on('kinect:pointCloudFrame', handlePointCloudFrame);
+    socket.on('kinect:colorFrame', handleColorFrame);
+    socket.emit('kinect:requestCachedFrames', {}, () => {});
+
+    return () => {
+      socket.off('kinect:pointCloudFrame', handlePointCloudFrame);
+      socket.off('kinect:colorFrame', handleColorFrame);
+    };
+  }, [socket]);
+
+  useEffect(() => () => {
+    if (colorUrl) URL.revokeObjectURL(colorUrl);
+  }, [colorUrl]);
+
+  const cooldownRemainingMs = Math.max(0, Number(status.captureCooldownUntil || 0) - nowMs);
+  const controlsDisabled = !status.enabled || status.busy || cooldownRemainingMs > 0;
+  const cooldownText = cooldownRemainingMs > 0 ? `${Math.ceil(cooldownRemainingMs / 1000)}s` : null;
+  const statusPill = useMemo(
+    () =>
+      buildStatusPill({
+        cooldownText,
+        enabled: status.enabled,
+        busy: status.busy,
+        lastError: status.lastError,
+      }),
+    [cooldownText, status.busy, status.enabled, status.lastError],
+  );
+  const visibleError = requestError || status.lastError;
+
+  const emitRequest = useCallback(
+    (eventName) => {
+      if (!socket || controlsDisabled) return;
+      setRequestError(null);
+      socket.emit(eventName, {}, (resp = {}) => {
+        if (resp.error) {
+          setRequestError(resp.error);
+        }
+      });
+    },
+    [controlsDisabled, socket],
+  );
+
+  const actions = (
+    <div className="flex flex-wrap items-center justify-end gap-0.5 text-[0.68rem] text-slate-400">
+      <span className={`inline-flex min-w-[3.7rem] justify-center rounded border px-1 py-0.5 text-xs font-semibold ${statusPill.className}`}>
+        {statusPill.label}
+      </span>
+      <div className="inline-flex overflow-hidden rounded border border-slate-700">
+        {['3d', 'image'].map((option) => (
+          <button
+            key={option}
+            type="button"
+            className={`px-1 py-0.5 ${activeView === option ? 'bg-slate-600 text-white' : 'bg-transparent text-slate-400 hover:text-white'}`}
+            onClick={() => setActiveView(option)}
+          >
+            {option === '3d' ? '3D' : 'Image'}
+          </button>
+        ))}
+      </div>
+      <button
+        type="button"
+        className="button-dark px-1 py-0.25"
+        disabled={controlsDisabled}
+        onClick={() => emitRequest('kinect:requestPointCloud')}
+      >
+        Request 3D
+      </button>
+      <button
+        type="button"
+        className="button-dark px-1 py-0.25"
+        disabled={controlsDisabled}
+        onClick={() => emitRequest('kinect:requestColorImage')}
+      >
+        Request Image
+      </button>
+    </div>
+  );
+
+  return (
+    <CardFrame title="Kinect Viewer" actions={actions} bodyClassName="space-y-0.5 p-0.5 text-sm">
+      <div className="aspect-[4/3] w-full overflow-hidden rounded bg-black">
+        {activeView === '3d' && pointCloudFrame?.buffer ? (
+          <PointCloudViewer frame={pointCloudFrame} />
+        ) : activeView === '3d' ? (
+          <div className="flex h-full items-center justify-center text-center text-sm text-slate-400">
+            Request a 3D frame
+          </div>
+        ) : colorUrl ? (
+          <img src={colorUrl} alt="Kinect image" className="h-full w-full object-contain" />
+        ) : (
+          <div className="flex h-full items-center justify-center text-center text-sm text-slate-400">
+            Request an image
+          </div>
+        )}
+      </div>
+      {visibleError ? (
+        <p className="m-0 break-words text-[0.7rem] text-red-300">{String(visibleError).toLowerCase()}</p>
+      ) : null}
+    </CardFrame>
+  );
+}

webui/src/components/KinectPanel/utils.js

@@ -0,0 +1,53 @@
+// Kinect Panel Utilities
+// Purpose: Keeps small normalization helpers out of the visual component files.
+// Scope: Handles session status defaults, socket binary payload conversion, and title-bar status pill state.
+export const EMPTY_KINECT_STATUS = {
+  enabled: false,
+  available: false,
+  busy: false,
+  captureCooldownUntil: 0,
+  lastError: null,
+  hasPointCloud: false,
+  hasColorImage: false,
+};
+
+export function normalizeKinectStatus(status) {
+  return {
+    ...EMPTY_KINECT_STATUS,
+    ...(status && typeof status === 'object' ? status : {}),
+  };
+}
+
+export function normalizeBinaryPayload(buffer) {
+  if (!buffer) return null;
+  if (buffer instanceof ArrayBuffer) return buffer;
+  if (ArrayBuffer.isView(buffer)) {
+    return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+  }
+  return null;
+}
+
+export function buildStatusPill({ cooldownText, enabled, busy, lastError }) {
+  if (cooldownText) {
+    return {
+      label: cooldownText,
+      className: 'border-red-400 bg-red-600 text-red-50',
+    };
+  }
+  if (!enabled || lastError) {
+    return {
+      label: 'Off',
+      className: 'border-red-400 bg-red-700 text-red-50',
+    };
+  }
+  if (busy) {
+    return {
+      label: '...',
+      className: 'border-amber-200 bg-amber-500 text-amber-950',
+    };
+  }
+  return {
+    label: 'Ready',
+    className: 'border-emerald-300 bg-emerald-600 text-emerald-50',
+  };
+}

webui/src/components/RightPaneTabs/index.jsx

@@ -2,6 +2,7 @@
 // Purpose: Defines the Right Pane Tabs module and the local helpers/components used in this file.
 // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
 import RoomCameraPanel from '../RoomCameraPanel/index.jsx';
+import KinectPanel from '../KinectPanel/index.jsx';
 import HomeAssistantControls from '../HomeAssistantControls/index.jsx';
 import SettingsPanel from '../SettingsPanel/index.jsx';
 import HelpPanel from '../HelpPanel/index.jsx';
@@ -196,6 +197,7 @@ export default function RightPaneTabs({ layout, onOpenHelpOverlay }) {
               <HomeAssistantControls />
               <ButtonBoxPanel />
               <RoomCameraPanel defaultOrientation="horizontal" panelId="rightpane-telemetry" />
+              <KinectPanel />
             </div>
           </TabPanel>
           <TabPanel id="vip" keepMounted>