legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 09:31:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

mesh um

Browse Source
This commit is contained in:
legop3
2026-06-07 04:09:35 -04:00
parent c636b41cbc
commit 4efc2be646
6 changed files with 115 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/public/assets/index-CYxH68Qf.js → server/public/assets/index-DlSNvZve.js
+10 -10
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Roomba Rover" />
<title>Roomba Rover</title> <title>Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-CYxH68Qf.js"></script> <script type="module" crossorigin src="/assets/index-DlSNvZve.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-Dzjs7qyo.css"> <link rel="stylesheet" crossorigin href="/assets/index-Dzjs7qyo.css">
</head> </head>
<body> <body>
server/src/services/kinectService/hardware.js
+1
View File
@@ -206,6 +206,7 @@ async function capturePointCloud() {
height: meta.height, height: meta.height,
pointCount: meta.pointCount, pointCount: meta.pointCount,
format: meta.format, format: meta.format,
grid: Boolean(meta.grid),
strideBytes: 16, strideBytes: 16,
rgbFrameAgeMs: meta.rgbFrameAgeMs, rgbFrameAgeMs: meta.rgbFrameAgeMs,
depthFrameAgeMs: meta.depthFrameAgeMs, depthFrameAgeMs: meta.depthFrameAgeMs,
server/src/services/kinectService/native/kinect_worker.cpp
+12 -9
View File
@@ -262,29 +262,32 @@ void handle_pointcloud(int id) {
}; };
// Registered depth aligns with RGB, so each valid depth pixel can become a // Registered depth aligns with RGB, so each valid depth pixel can become a
// colored point without an additional calibration lookup. Invalid zero-depth // colored vertex without an additional calibration lookup. Keep one fixed
// pixels are skipped to keep the payload and browser point count smaller. // record for every 640x480 pixel, including invalid depth pixels, because the
// browser needs the original image grid to decide which neighboring vertices
// can be connected into triangles. Invalid pixels get alpha 0 and zeroed
// coordinates; the viewer skips them when building the surface mesh.
for (int y = 0; y < kHeight; y += 1) { for (int y = 0; y < kHeight; y += 1) {
for (int x = 0; x < kWidth; x += 1) { for (int x = 0; x < kWidth; x += 1) {
const int idx = y * kWidth + x; const int idx = y * kWidth + x;
const uint16_t z_mm = depth[idx]; const uint16_t z_mm = depth[idx];
if (z_mm == 0) continue; const bool valid = z_mm != 0;
const float z = static_cast<float>(z_mm) / 1000.0f; const float z = valid ? static_cast<float>(z_mm) / 1000.0f : 0.0f;
const float world_x = (static_cast<float>(x) - center_x) * z / focal_x; const float world_x = valid ? (static_cast<float>(x) - center_x) * z / focal_x : 0.0f;
const float world_y = -(static_cast<float>(y) - center_y) * z / focal_y; const float world_y = valid ? -(static_cast<float>(y) - center_y) * z / focal_y : 0.0f;
append_float(world_x); append_float(world_x);
append_float(world_y); append_float(world_y);
append_float(z); append_float(z);
payload.push_back(rgb[idx * 3 + 0]); payload.push_back(rgb[idx * 3 + 0]);
payload.push_back(rgb[idx * 3 + 1]); payload.push_back(rgb[idx * 3 + 1]);
payload.push_back(rgb[idx * 3 + 2]); payload.push_back(rgb[idx * 3 + 2]);
payload.push_back(255); payload.push_back(valid ? 255 : 0);
point_count += 1; if (valid) point_count += 1;
} }
} }
std::ostringstream meta; std::ostringstream meta;
meta << ",\"kind\":\"pointCloud\",\"format\":\"xyzrgb-f32-u8\",\"width\":" << kWidth meta << ",\"kind\":\"pointCloud\",\"format\":\"xyzrgb-grid-f32-u8\",\"grid\":true,\"width\":" << kWidth
<< ",\"height\":" << kHeight << ",\"pointCount\":" << point_count << ",\"height\":" << kHeight << ",\"pointCount\":" << point_count
<< ",\"rgbFrameAgeMs\":" << rgb_age << ",\"depthFrameAgeMs\":" << depth_age; << ",\"rgbFrameAgeMs\":" << rgb_age << ",\"depthFrameAgeMs\":" << depth_age;
write_packet(id, meta.str(), payload); write_packet(id, meta.str(), payload);
server/src/services/kinectService/socketGateway.js
+18 -2
View File
@@ -68,6 +68,9 @@ function registerKinectSocketGateway({ config, hardware }) {
} }
function sendCachedFrames(socket) { function sendCachedFrames(socket) {
if (!passesMode(socket)) {
return;
}
// Cached-frame replay gives newly opened tabs the latest room snapshot // Cached-frame replay gives newly opened tabs the latest room snapshot
// without starting a new Kinect capture or spending upload continuously. // without starting a new Kinect capture or spending upload continuously.
if (lastPointCloud?.buffer) { if (lastPointCloud?.buffer) {
@@ -78,6 +81,19 @@ function registerKinectSocketGateway({ config, hardware }) {
} }
} }
function broadcastFrame(eventName, meta, buffer) {
// Kinect frames are room-privacy-sensitive, especially in lockdown mode.
// Do not use io.emit here: every frame must be checked against the current
// mode because lockdown is explicitly a privacy mode where only lockdown
// admins should receive camera-like data.
io.sockets.sockets.forEach((socket) => {
if (!passesMode(socket)) {
return;
}
socket.emit(eventName, meta, buffer);
});
}
function rejectDisabled() { function rejectDisabled() {
if (!settings.enabled) { if (!settings.enabled) {
return { error: 'kinect service is disabled' }; return { error: 'kinect service is disabled' };
@@ -143,10 +159,10 @@ function registerKinectSocketGateway({ config, hardware }) {
}; };
if (kind === 'pointCloud') { if (kind === 'pointCloud') {
lastPointCloud = { meta, buffer: capture.buffer }; lastPointCloud = { meta, buffer: capture.buffer };
io.emit('kinect:pointCloudFrame', meta, capture.buffer); broadcastFrame('kinect:pointCloudFrame', meta, capture.buffer);
} else { } else {
lastColorImage = { meta, buffer: capture.buffer }; lastColorImage = { meta, buffer: capture.buffer };
io.emit('kinect:colorFrame', meta, capture.buffer); broadcastFrame('kinect:colorFrame', meta, capture.buffer);
} }
logger.info('Kinect capture broadcast', { logger.info('Kinect capture broadcast', {
kind, kind,
webui/src/components/KinectPanel/PointCloudViewer.jsx
+69 -22
View File
@@ -8,7 +8,7 @@ export default function PointCloudViewer({ frame }) {
const rendererRef = useRef(null); const rendererRef = useRef(null);
const cameraRef = useRef(null); const cameraRef = useRef(null);
const sceneRef = useRef(null); const sceneRef = useRef(null);
const pointsRef = useRef(null); const objectRef = useRef(null);
const controlsRef = useRef(null); const controlsRef = useRef(null);
const threeRef = useRef(null); const threeRef = useRef(null);
const visibleRef = useRef(false); const visibleRef = useRef(false);
@@ -22,6 +22,15 @@ export default function PointCloudViewer({ frame }) {
renderer.render(scene, camera); renderer.render(scene, camera);
}, []); }, []);
const disposeRenderedObject = useCallback(() => {
const object = objectRef.current;
if (!object) return;
sceneRef.current?.remove(object);
object.geometry?.dispose();
object.material?.dispose();
objectRef.current = null;
}, []);
const rebuildGeometry = useCallback(() => { const rebuildGeometry = useCallback(() => {
const scene = sceneRef.current; const scene = sceneRef.current;
const currentFrame = frameRef.current; const currentFrame = frameRef.current;
@@ -33,44 +42,87 @@ export default function PointCloudViewer({ frame }) {
if (!pointCount || strideBytes < 16) return; if (!pointCount || strideBytes < 16) return;
const view = new DataView(currentFrame.buffer); const view = new DataView(currentFrame.buffer);
const positions = new Float32Array(pointCount * 3); const width = Number(currentFrame.meta?.width) || 0;
const colors = new Float32Array(pointCount * 3); const height = Number(currentFrame.meta?.height) || 0;
const gridPointCount = width * height;
const isGridFrame = Boolean(currentFrame.meta?.grid) && gridPointCount > 0;
const vertexCount = isGridFrame ? gridPointCount : pointCount;
const positions = new Float32Array(vertexCount * 3);
const colors = new Float32Array(vertexCount * 3);
const valid = isGridFrame ? new Uint8Array(vertexCount) : null;
const zValues = isGridFrame ? new Float32Array(vertexCount) : null;
// The server sends x/y/z as little-endian floats followed by rgba bytes. // The server sends x/y/z as little-endian floats followed by rgba bytes.
// Building typed arrays only when the canvas is visible keeps expensive // Building typed arrays only when the canvas is visible keeps expensive
// browser-side point conversion from happening while the card is off-screen. // browser-side point conversion from happening while the card is off-screen.
for (let index = 0; index < pointCount; index += 1) { for (let index = 0; index < vertexCount; index += 1) {
const source = index * strideBytes; const source = index * strideBytes;
const target = index * 3; const target = index * 3;
positions[target + 0] = view.getFloat32(source + 0, true); positions[target + 0] = view.getFloat32(source + 0, true);
positions[target + 1] = view.getFloat32(source + 4, true); positions[target + 1] = view.getFloat32(source + 4, true);
positions[target + 2] = -view.getFloat32(source + 8, true); const z = view.getFloat32(source + 8, true);
positions[target + 2] = -z;
colors[target + 0] = view.getUint8(source + 12) / 255; colors[target + 0] = view.getUint8(source + 12) / 255;
colors[target + 1] = view.getUint8(source + 13) / 255; colors[target + 1] = view.getUint8(source + 13) / 255;
colors[target + 2] = view.getUint8(source + 14) / 255; colors[target + 2] = view.getUint8(source + 14) / 255;
if (isGridFrame) {
valid[index] = view.getUint8(source + 15) > 0 ? 1 : 0;
zValues[index] = z;
}
} }
const geometry = new THREE.BufferGeometry(); const geometry = new THREE.BufferGeometry();
geometry.setAttribute('position', new THREE.BufferAttribute(positions, 3)); geometry.setAttribute('position', new THREE.BufferAttribute(positions, 3));
geometry.setAttribute('color', new THREE.BufferAttribute(colors, 3)); geometry.setAttribute('color', new THREE.BufferAttribute(colors, 3));
geometry.computeBoundingSphere();
let object = null;
if (isGridFrame) {
const indices = [];
const maxDepthStepMeters = 0.12;
const canConnect = (a, b, c) => {
if (!valid[a] || !valid[b] || !valid[c]) return false;
const minZ = Math.min(zValues[a], zValues[b], zValues[c]);
const maxZ = Math.max(zValues[a], zValues[b], zValues[c]);
return maxZ - minZ <= maxDepthStepMeters;
};
// Kinect depth is a regular image. Each 2x2 pixel cell can become two
// triangles, but only when all vertices are valid and close in depth. The
// depth-step check prevents the mesh from drawing sheets across object
// edges, missing-depth holes, or foreground/background gaps.
for (let y = 0; y < height - 1; y += 1) {
for (let x = 0; x < width - 1; x += 1) {
const a = y * width + x;
const b = a + 1;
const c = a + width;
const d = c + 1;
if (canConnect(a, c, b)) indices.push(a, c, b);
if (canConnect(b, c, d)) indices.push(b, c, d);
}
}
geometry.setIndex(indices);
geometry.computeVertexNormals();
const material = new THREE.MeshBasicMaterial({
vertexColors: true,
side: THREE.DoubleSide,
});
object = new THREE.Mesh(geometry, material);
} else {
const material = new THREE.PointsMaterial({ const material = new THREE.PointsMaterial({
size: 0.018, size: 0.018,
vertexColors: true, vertexColors: true,
sizeAttenuation: true, sizeAttenuation: true,
}); });
const points = new THREE.Points(geometry, material); object = new THREE.Points(geometry, material);
if (pointsRef.current) {
scene.remove(pointsRef.current);
pointsRef.current.geometry.dispose();
pointsRef.current.material.dispose();
} }
pointsRef.current = points;
scene.add(points); geometry.computeBoundingSphere();
disposeRenderedObject();
objectRef.current = object;
scene.add(object);
renderOnce(); renderOnce();
}, [renderOnce]); }, [disposeRenderedObject, renderOnce]);
useEffect(() => { useEffect(() => {
frameRef.current = frame; frameRef.current = frame;
@@ -153,12 +205,7 @@ export default function PointCloudViewer({ frame }) {
resizeObserver?.disconnect(); resizeObserver?.disconnect();
controlsRef.current?.removeEventListener('change', renderOnce); controlsRef.current?.removeEventListener('change', renderOnce);
controlsRef.current?.dispose(); controlsRef.current?.dispose();
if (pointsRef.current) { disposeRenderedObject();
sceneRef.current?.remove(pointsRef.current);
pointsRef.current.geometry.dispose();
pointsRef.current.material.dispose();
pointsRef.current = null;
}
rendererRef.current?.dispose(); rendererRef.current?.dispose();
rendererRef.current?.domElement?.remove(); rendererRef.current?.domElement?.remove();
sceneRef.current = null; sceneRef.current = null;
@@ -167,7 +214,7 @@ export default function PointCloudViewer({ frame }) {
controlsRef.current = null; controlsRef.current = null;
threeRef.current = null; threeRef.current = null;
}; };
}, [rebuildGeometry, renderOnce]); }, [disposeRenderedObject, rebuildGeometry, renderOnce]);
return <div ref={hostRef} className="h-full w-full" />; return <div ref={hostRef} className="h-full w-full" />;
} }