better video ptz stuf

This commit is contained in:
legop3
2026-07-11 18:25:30 -04:00
parent e4ada54cf4
commit 0bb3f89472
9 changed files with 306 additions and 174 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -78,7 +78,7 @@
<script defer src="https://analytics.otter.land/script.js" data-website-id="82dd56a5-db44-4279-bd1e-a4d9fee39af7" data-domains="rover.otter.land"></script>
<script defer src="https://analytics.otter.land/recorder.js" data-website-id="82dd56a5-db44-4279-bd1e-a4d9fee39af7" data-domains="rover.otter.land" data-sample-rate="0.15" data-mask-level="moderate" data-max-duration="300000"></script>
<title>Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-Bi6EaYls.js"></script>
<script type="module" crossorigin src="/assets/index-B8x6P1kX.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-MAURNhur.css">
</head>
<body>
@@ -7,6 +7,7 @@ const { issueCommand } = require('../commandService');
const homeAssistantService = require('../homeAssistantService');
const neatoService = require('../neatoService');
const liftService = require('../liftService');
const ptzCameraService = require('../ptzCameraService');
const {
HEADLIGHT_DISABLE_ACTION,
LASER_DISABLE_ACTION,
@@ -103,6 +104,17 @@ async function disableAllRoverLasers() {
return { action: 'disableRoverLasers', attempted, failed };
}
async function disablePtzEmitters() {
/*
The PTZ camera has its own light APIs and ownership rules, so the idle
service delegates the actual Reolink calls to ptzCameraService instead of
pretending they are rover commands. This keeps idleService responsible only
for "idle fired; run cleanup actions" and keeps camera-specific payload
details beside the rest of the PTZ integration.
*/
return ptzCameraService.disableEmittersForIdle();
}
async function sendNeatoHome() {
try {
await neatoService.sendHome();
@@ -126,6 +138,7 @@ const idleActions = [
// dockAllRovers,
disableAllRoverHeadlights,
disableAllRoverLasers,
disablePtzEmitters,
sendNeatoHome,
raiseLift,
];
@@ -756,6 +756,81 @@ async function setIr(socket, payload = {}) {
});
}
async function disableEmittersForIdle() {
/*
Idle cleanup is a server-owned safety action, not a user control action, so
it intentionally does not go through requireOperator(). If nobody is using
the camera, the system still needs a way to leave every camera-side emitter
in a known off state.
*/
if (!enabled) {
return { action: 'disablePtzEmitters', skipped: true, reason: 'ptzDisabled' };
}
await initialize();
if (!state.initialized) {
return {
action: 'disablePtzEmitters',
success: false,
error: state.error || 'PTZ camera is not ready',
};
}
return serializeVendorState(async () => {
const client = await ensureReolinkClient();
const lightPayload = { channel: 0, state: spotlightCameraStateForLogicalOn(false) };
const irPayload = { channel: 0, state: normalizeIrState('off') };
const failures = [];
/*
Set the public state before the API calls finish so the UI immediately
reflects the idle policy. If a camera call fails, the result still records
that failure and the next vendor refresh can correct the optimistic state.
*/
state.light = normalizeSpotlightState({
...(state.light || {}),
...lightPayload,
on: false,
});
state.ir = {
...(state.ir || {}),
...irPayload,
};
emitChange('idle-emitters-off-pending');
try {
await client.api('SetWhiteLed', { WhiteLed: lightPayload });
} catch (err) {
failures.push({ control: 'spotlight', error: err.message });
}
try {
await client.api('SetIrLights', { IrLights: irPayload });
} catch (err) {
failures.push({ control: 'ir', error: err.message });
}
/*
Read back once after the writes so stale optimistic state does not linger
forever. The existing spotlight button path delays verification because it
is user-facing and frequently toggled; idle fires rarely, so one ordered
refresh keeps the final state simple.
*/
try {
await refreshVendorState();
} catch (err) {
failures.push({ control: 'refresh', error: err.message });
}
emitChange('idle-emitters-off');
return {
action: 'disablePtzEmitters',
success: failures.length === 0,
failures,
};
});
}
function canRequestLiveVideo(socket) {
if (!enabled || !passesMode(socket)) return false;
if (state.operatorSocketId === socket?.id) return true;
@@ -973,6 +1048,7 @@ module.exports = {
ptzCameraEvents: events,
getPublicState,
canRequestLiveVideo,
disableEmittersForIdle,
getReplaySource: () => enabled && isReplayEnabled()
? { type: 'ptz', id: PTZ_CAMERA_ID, label: cameraConfig.name || 'PTZ Camera' }
: null,
+177
View File
@@ -0,0 +1,177 @@
// PTZ Live Video
// Purpose: Plays the single PTZ camera WHEP stream with the same fresh-session retry loop used by rover video.
// Scope: Owns browser-side WHEP playback/retry only; server authorization and snapshot fallback policy stay outside.
import { useCallback, useEffect, useRef, useState } from 'react';
import { useVideoRequests } from '../../hooks/useVideoRequests.js';
import { WhepPlayer } from '../../lib/whepPlayer.js';
import { RESTART_DELAY_MS } from '../RoverMediaPlayer/constants.js';
export const PTZ_CAMERA_ID = 'ptz-camera';
const PTZ_AUDIO_RETRY_MS = 1000;
const TERMINAL_WHEP_STATES = new Set(['error', 'failed', 'disconnected', 'closed', 'stopped']);
const AUTHORIZATION_ERROR_RE = /not authorized/i;
function isAuthorizationError(error) {
return AUTHORIZATION_ERROR_RE.test(String(error || ''));
}
export default function PtzLiveVideo({
enabled = true,
startMuted = true,
className = 'relative h-full w-full bg-black',
videoClassName = 'h-full w-full object-contain',
statusClassName = 'pointer-events-none absolute bottom-2 left-2 rounded bg-black/70 px-2 py-1 text-xs text-slate-100',
label = null,
labelClassName = 'pointer-events-none absolute left-0 top-0 bg-black/70 px-1 py-0.5 text-xs font-semibold text-white',
fallback = null,
}) {
const videoRef = useRef(null);
const retryTimerRef = useRef(null);
const playTimerRef = useRef(null);
const [status, setStatus] = useState('idle');
const [detail, setDetail] = useState(null);
const [restartToken, setRestartToken] = useState(0);
const sources = useVideoRequests(
[{ type: 'ptz', id: PTZ_CAMERA_ID, key: PTZ_CAMERA_ID }],
{ enabled, version: restartToken },
);
const source = sources[PTZ_CAMERA_ID] || null;
const shouldUseFallback = Boolean(source?.error && isAuthorizationError(source.error));
const scheduleRestart = useCallback(() => {
if (!enabled || shouldUseFallback) return;
/*
WHEP sessions are one-shot browser/server negotiations. When the camera
reboots, the old PeerConnection and token can look alive enough to keep a
black element on screen, but they are not useful anymore. Bumping this
token forces useVideoRequests to ask the server for a new MediaMTX auth
session before creating the next WhepPlayer.
*/
clearTimeout(retryTimerRef.current);
retryTimerRef.current = setTimeout(() => {
retryTimerRef.current = null;
setRestartToken(Date.now());
}, RESTART_DELAY_MS);
}, [enabled, shouldUseFallback]);
useEffect(() => {
if (!enabled || shouldUseFallback || !source?.url || !videoRef.current) return undefined;
let active = true;
const player = new WhepPlayer({
url: source.url,
token: source.token,
video: videoRef.current,
startMuted,
onStatus: (nextStatus, info) => {
if (!active) return;
const normalized = String(nextStatus || '').toLowerCase();
setStatus(nextStatus || 'unknown');
setDetail(info || null);
if (TERMINAL_WHEP_STATES.has(normalized)) {
scheduleRestart();
}
},
});
player.start().catch((err) => {
if (!active) return;
setStatus('error');
setDetail(err.message || 'WHEP start failed');
scheduleRestart();
});
return () => {
/*
Mark inactive before stop() because WhepPlayer reports "stopped" during
normal cleanup. Cleanup-driven stops should not immediately schedule the
next retry; only the replacement effect should own the new connection.
*/
active = false;
player.stop();
};
}, [enabled, scheduleRestart, shouldUseFallback, source?.token, source?.url, startMuted]);
useEffect(() => {
const video = videoRef.current;
if (!enabled || shouldUseFallback || !source?.url || !video) return undefined;
const handleEnded = () => {
setStatus('stopped');
setDetail('ended');
scheduleRestart();
};
const handleError = () => {
setStatus('error');
setDetail(video.error?.message || 'video element error');
scheduleRestart();
};
video.addEventListener('ended', handleEnded);
video.addEventListener('error', handleError);
return () => {
video.removeEventListener('ended', handleEnded);
video.removeEventListener('error', handleError);
};
}, [enabled, scheduleRestart, shouldUseFallback, source?.url]);
useEffect(() => {
if (status === 'stopped' && source?.url && !shouldUseFallback) {
scheduleRestart();
}
}, [scheduleRestart, shouldUseFallback, source?.url, status]);
useEffect(() => {
const video = videoRef.current;
if (!enabled || shouldUseFallback || startMuted || !source?.url || !video) {
if (playTimerRef.current) clearInterval(playTimerRef.current);
playTimerRef.current = null;
return undefined;
}
/*
PTZ carries inline Opus audio. When the operator opened the camera from a
user gesture, keep retrying audible playback so browser autoplay timing
does not leave the element permanently muted after a reconnect.
*/
const attemptPlay = () => {
const target = videoRef.current;
if (!target) return;
target.muted = false;
if (!target.paused && !target.ended) return;
target.play().catch(() => {});
};
attemptPlay();
playTimerRef.current = setInterval(attemptPlay, PTZ_AUDIO_RETRY_MS);
return () => {
if (playTimerRef.current) clearInterval(playTimerRef.current);
playTimerRef.current = null;
};
}, [enabled, shouldUseFallback, source?.url, startMuted, status]);
useEffect(() => () => {
if (retryTimerRef.current) clearTimeout(retryTimerRef.current);
if (playTimerRef.current) clearInterval(playTimerRef.current);
}, []);
if (shouldUseFallback && typeof fallback === 'function') {
return fallback({ source, status, detail });
}
const displayStatus = source?.error || detail || status;
return (
<div className={className}>
{source?.url ? (
<video ref={videoRef} className={videoClassName} playsInline autoPlay muted={startMuted} />
) : (
<div className="flex h-full w-full items-center justify-center text-xs text-slate-400">
{source?.error || 'Waiting for PTZ video...'}
</div>
)}
{label ? <div className={labelClassName}>{label}</div> : null}
<div className={statusClassName}>{displayStatus}</div>
</div>
);
}
+6 -113
View File
@@ -6,18 +6,16 @@ import { createPortal } from 'react-dom';
import CardFrame from '../CardFrame/index.jsx';
import GPIOToggleControl from '../GPIOToggleControl/index.jsx';
import ControlPadPanel from '../MobileControls/ControlPadPanel.jsx';
import PtzLiveVideo from '../PtzLiveVideo/index.jsx';
import ReplaySourcesPanel from '../ReplaySourcesPanel/index.jsx';
import KeyPill from './VipAudioUploadCard/KeyPill.jsx';
import { useSessionActions, useSessionSelector } from '../../context/SessionContext.jsx';
import { useControlSelector } from '../../controls/index.js';
import { formatKeyLabel } from '../../controls/keymapUtils.js';
import { usePtzCameraSnapshot } from '../../hooks/usePtzCameraSnapshot.js';
import { useVideoRequests } from '../../hooks/useVideoRequests.js';
import { WhepPlayer } from '../../lib/whepPlayer.js';
import { isFeatureEnabled } from '../../lib/features.js';
const PTZ_CAMERA_ID = 'ptz-camera';
const PTZ_AUDIO_RETRY_MS = 1000;
const PTZ_ZOOM_SPEED = 0.55;
function formatRemaining(deadline) {
@@ -147,115 +145,6 @@ function PtzStatePanel({ ptz, onClose, onRelease, releaseDisabled = false }) {
);
}
function PtzLiveVideo({ enabled }) {
const videoRef = useRef(null);
const playerRef = useRef(null);
const retryTimerRef = useRef(null);
const playTimerRef = useRef(null);
const [status, setStatus] = useState('idle');
const [retryVersion, setRetryVersion] = useState(0);
const sources = useVideoRequests(
[{ type: 'ptz', id: PTZ_CAMERA_ID, key: PTZ_CAMERA_ID }],
{ enabled, version: retryVersion },
);
const source = sources[PTZ_CAMERA_ID] || null;
const scheduleRetry = useCallback(() => {
if (!enabled || retryTimerRef.current) return;
/*
A failed WHEP POST consumes the short-lived video token and leaves the
PeerConnection in a terminal state. Requesting a fresh server session is
the simplest reliable retry path, and it matches how rover playback gets
a new authorization token after reconnects.
*/
retryTimerRef.current = setTimeout(() => {
retryTimerRef.current = null;
setRetryVersion((value) => value + 1);
}, 1500);
}, [enabled]);
useEffect(() => {
if (!enabled || !source?.url || !videoRef.current) return undefined;
/*
WhepPlayer already owns PeerConnection setup, low-latency hints, auth
headers, and cleanup for rover video. Reusing it keeps PTZ video on the
same MediaMTX browser path as the rest of the app.
*/
const player = new WhepPlayer({
url: source.url,
token: source.token,
video: videoRef.current,
startMuted: false,
onStatus: (nextStatus) => {
setStatus(nextStatus);
if (['error', 'failed', 'disconnected', 'closed'].includes(String(nextStatus || '').toLowerCase())) {
scheduleRetry();
}
},
});
playerRef.current = player;
player.start().catch((err) => {
setStatus(err.message || 'error');
scheduleRetry();
});
return () => {
player.stop();
playerRef.current = null;
};
}, [enabled, scheduleRetry, source?.token, source?.url]);
useEffect(() => {
const video = videoRef.current;
if (!enabled || !source?.url || !video) {
if (playTimerRef.current) clearInterval(playTimerRef.current);
playTimerRef.current = null;
return undefined;
}
/*
The server now publishes inline Opus audio on the PTZ WHEP stream. The
shared WHEP helper starts playback once, but browsers can still reject or
pause audible media depending on the exact timing of the fullscreen/user
gesture. Retry the same element with muted=false so unmuting is not a
manual DevTools-only operation.
*/
const attemptPlay = () => {
const target = videoRef.current;
if (!target) return;
target.muted = false;
if (!target.paused && !target.ended) return;
target.play().catch(() => {});
};
attemptPlay();
playTimerRef.current = setInterval(attemptPlay, PTZ_AUDIO_RETRY_MS);
return () => {
if (playTimerRef.current) clearInterval(playTimerRef.current);
playTimerRef.current = null;
};
}, [enabled, source?.url, status]);
useEffect(() => {
return () => {
if (retryTimerRef.current) {
clearTimeout(retryTimerRef.current);
retryTimerRef.current = null;
}
if (playTimerRef.current) {
clearInterval(playTimerRef.current);
playTimerRef.current = null;
}
};
}, []);
return (
<div className="relative h-full w-full bg-black">
<video ref={videoRef} className="h-full w-full object-contain" playsInline autoPlay />
<div className="pointer-events-none absolute bottom-2 left-2 rounded bg-black/70 px-2 py-1 text-xs text-slate-100">
{source?.error || status}
</div>
</div>
);
}
function PtzLightingControls({ ptz, disabled = false }) {
const { ptzSpotlight, ptzIr } = useSessionActions();
const [busy, setBusy] = useState('');
@@ -500,7 +389,11 @@ function PtzController({ open, onClose, layout = 'desktop' }) {
bodyClassName={`grid h-full min-h-0 overflow-hidden ${sidebarWidthClass}`}
>
<main className="relative min-h-0 min-w-0 bg-black">
{isOperator ? <PtzLiveVideo enabled /> : <PtzSnapshotPreview feed={snapshot} label={ptz?.name || 'PTZ Camera'} />}
{isOperator ? (
<PtzLiveVideo enabled startMuted={false} />
) : (
<PtzSnapshotPreview feed={snapshot} label={ptz?.name || 'PTZ Camera'} />
)}
</main>
<aside className="flex h-full min-h-0 items-stretch overflow-hidden border-l border-neutral-600 bg-neutral-950 text-sm">
<div className="flex min-h-0 w-full flex-col gap-0.5 overflow-y-auto p-0.5">
@@ -2,13 +2,9 @@
// Purpose: Adds the single room PTZ camera to the spectator rover grid.
// Scope: Uses live WHEP only when the server authorizes this socket; otherwise
// falls back to the PTZ snapshot feed that remote spectators are allowed to see.
import { useEffect, useRef, useState } from 'react';
import PtzLiveVideo from '../../../components/PtzLiveVideo/index.jsx';
import { useSessionSelector } from '../../../context/SessionContext.jsx';
import { usePtzCameraSnapshot } from '../../../hooks/usePtzCameraSnapshot.js';
import { useVideoRequests } from '../../../hooks/useVideoRequests.js';
import { WhepPlayer } from '../../../lib/whepPlayer.js';
const PTZ_CAMERA_ID = 'ptz-camera';
function formatRemaining(deadline) {
const remaining = Math.max(0, Math.ceil((Number(deadline || 0) - Date.now()) / 1000));
@@ -44,63 +40,40 @@ function InfoRow({ label, value, tone = '' }) {
);
}
function PtzLiveOrSnapshot({ label }) {
const videoRef = useRef(null);
const playerRef = useRef(null);
const [liveStatus, setLiveStatus] = useState('connecting');
const sources = useVideoRequests(
[{ type: 'ptz', id: PTZ_CAMERA_ID, key: PTZ_CAMERA_ID }],
{ enabled: true },
);
const source = sources[PTZ_CAMERA_ID] || null;
const snapshot = usePtzCameraSnapshot({ enabled: Boolean(source?.error || !source?.url) });
const useSnapshot = Boolean(source?.error || !source?.url);
useEffect(() => {
if (useSnapshot || !source?.url || !videoRef.current) return undefined;
/*
Spectator live access is decided by the server's video session policy.
Local spectators get a WHEP URL, while remote spectators receive an error
here and automatically fall back to the low-bandwidth snapshot feed.
*/
const player = new WhepPlayer({
url: source.url,
token: source.token,
video: videoRef.current,
startMuted: true,
onStatus: setLiveStatus,
});
playerRef.current = player;
player.start().catch((err) => setLiveStatus(err.message || 'error'));
return () => {
player.stop();
playerRef.current = null;
};
}, [source?.token, source?.url, useSnapshot]);
function PtzSnapshotFallback({ label, source }) {
const snapshot = usePtzCameraSnapshot({ enabled: true });
return (
<div className="relative aspect-video w-full overflow-hidden rounded bg-black">
{useSnapshot ? (
snapshot?.objectUrl ? (
<img src={snapshot.objectUrl} alt={label} className="h-full w-full object-cover" />
) : (
<div className="flex h-full w-full items-center justify-center text-xs text-slate-400">
{snapshot?.error || source?.error || 'Waiting for PTZ snapshot...'}
</div>
)
{snapshot?.objectUrl ? (
<img src={snapshot.objectUrl} alt={label} className="h-full w-full object-cover" />
) : (
<video ref={videoRef} className="h-full w-full object-contain" playsInline autoPlay muted />
<div className="flex h-full w-full items-center justify-center text-xs text-slate-400">
{snapshot?.error || source?.error || 'Waiting for PTZ snapshot...'}
</div>
)}
<div className="pointer-events-none absolute left-0 top-0 bg-black/70 px-1 py-0.5 text-xs font-semibold text-white">
{label}
</div>
<div className="pointer-events-none absolute bottom-0 left-0 m-1 rounded bg-black/70 px-1 py-0.5 text-[0.7rem] text-slate-100">
{useSnapshot ? snapshot?.status || 'snapshot' : liveStatus}
{snapshot?.status || 'snapshot'}
</div>
</div>
);
}
function PtzLiveOrSnapshot({ label }) {
return (
<PtzLiveVideo
enabled
startMuted
label={label}
className="relative aspect-video w-full overflow-hidden rounded bg-black"
statusClassName="pointer-events-none absolute bottom-0 left-0 m-1 rounded bg-black/70 px-1 py-0.5 text-[0.7rem] text-slate-100"
fallback={({ source }) => <PtzSnapshotFallback label={label} source={source} />}
/>
);
}
export default function PtzSpectatorCard() {
const ptz = useSessionSelector((state) => state.session?.ptzCamera || null);
if (!ptz?.enabled) return null;