diff --git a/server/src/services/ptzCameraService/index.js b/server/src/services/ptzCameraService/index.js index dbdd5f9b..43330ca3 100644 --- a/server/src/services/ptzCameraService/index.js +++ b/server/src/services/ptzCameraService/index.js @@ -33,6 +33,8 @@ const SNAPSHOT_POLL_MS = 300; const SNAPSHOT_STREAM_INTERVAL_MS = 2000; const SPOTLIGHT_VERIFY_DELAY_MS = 1200; const PUBLISHER_STDERR_SYNC_MS = 10000; +const PUBLISHER_RTSP_TIMEOUT_US = 10000000; +const REOLINK_API_RETRY_MS = 1000; const events = new EventEmitter(); const config = loadConfig(); @@ -68,6 +70,15 @@ const state = { progress: null, lastEvent: 'idle', }, + reolinkApi: { + connected: false, + connecting: false, + retryCount: 0, + retryAt: null, + lastError: null, + lastConnectedAt: null, + lastEvent: 'idle', + }, }; let onvifCam = null; @@ -80,6 +91,7 @@ let publisherStderrSyncTimer = null; let snapshotTimer = null; let spotlightVerifyTimer = null; let vendorStatePromise = Promise.resolve(); +let reolinkApiLogNextAt = 0; let lastSnapshotState = null; const snapshotSubscribers = new Map(); const socketSnapshotSubscriptions = new Map(); @@ -110,6 +122,20 @@ function updatePublisherState(patch = {}, reason = 'publisher') { emitChange(reason); } +function updateReolinkApiState(patch = {}, reason = 'reolink-api') { + state.reolinkApi = { + ...(state.reolinkApi || {}), + ...patch, + }; + emitChange(reason); +} + +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + function parsePublisherProgressLine(line) { /* ffmpeg's "-progress pipe:2" emits simple key=value telemetry on stderr. @@ -286,6 +312,7 @@ function getPublicState(socket = null) { presets: state.presets, presetsError: state.presetsError, publisher: state.publisher, + reolinkApi: state.reolinkApi, isOperator: Boolean(socketId && state.operatorSocketId === socketId), queuedPosition: socketId ? state.queue.indexOf(socketId) + 1 || null : null, canUse: socket ? canUsePtzFeature(socket) : false, @@ -451,6 +478,31 @@ function stopPublisher() { }, 'publisher-stop'); } +function schedulePublisherRestart(reason = 'publisher-restart') { + /* + The publisher's recovery rule is intentionally simple: ffmpeg owns the RTSP + connection, and this service starts a fresh process whenever that connection + causes ffmpeg to exit. Clearing any existing timer first prevents a burst of + quick exits from scheduling multiple competing replacement publishers. + */ + if (!enabled || !state.rtspUri) return null; + if (publisherRestartTimer) { + clearTimeout(publisherRestartTimer); + publisherRestartTimer = null; + } + const restartAt = Date.now() + 1500; + publisherRestartTimer = setTimeout(() => { + publisherRestartTimer = null; + startPublisher(); + }, 1500); + updatePublisherState({ + restartAt, + restartCount: Number(state.publisher?.restartCount || 0) + 1, + lastEvent: reason, + }, 'publisher-restart-scheduled'); + return restartAt; +} + function startPublisher() { if (!enabled || !state.rtspUri || publisherProcess) return; const input = addCredentialsToRtsp(state.rtspUri); @@ -481,6 +533,13 @@ function startPublisher() { safer latency knob is to keep the encoder light and avoid building delay inside x264 itself. + The camera can restart while ffmpeg keeps its old TCP/RTSP session open and + continues publishing a useless black output stream. The timeout options are + input-side failure detectors: when the RTSP socket stops producing usable + reads for long enough, ffmpeg should exit instead of staying attached to the + dead session. The existing exit handler then starts a new process, which is + the part that creates a fresh RTSP connection after the camera comes back. + The mpegts muxer can also hold packets briefly before writing them to SRT. flush_packets/muxdelay/muxpreload are output-side latency knobs; they do not ask the camera or demuxer to discard frames, so they are a safer next step @@ -501,6 +560,10 @@ function startPublisher() { 'low_delay', '-rtsp_transport', 'tcp', + '-rw_timeout', + String(PUBLISHER_RTSP_TIMEOUT_US), + '-timeout', + String(PUBLISHER_RTSP_TIMEOUT_US), '-i', input, '-map', @@ -571,24 +634,42 @@ function startPublisher() { running: false, pid: null, restartAt, - restartCount: Number(state.publisher?.restartCount || 0) + (restartAt ? 1 : 0), exitCode: code, exitSignal: signal, exitedAt: Date.now(), lastEvent: restartAt ? 'restarting' : 'exited', }, 'publisher-exit'); - if (enabled && state.rtspUri) { - publisherRestartTimer = setTimeout(() => { - publisherRestartTimer = null; - startPublisher(); - }, 1500); - } + if (restartAt) schedulePublisherRestart('restarting'); }); logger.info('Started PTZ stream publisher', { streamPath: PTZ_STREAM_PATH, encoder: 'libx264' }); } -async function ensureReolinkClient() { - if (reolinkClient) return reolinkClient; +function getErrorMessage(err) { + return err?.message || String(err || 'unknown error'); +} + +async function closeReolinkClient(client, reason = 'reset') { + /* + The Reolink SDK keeps a long-mode login session and may also own background + resources internally. close() is documented as idempotent, so use it as the + preferred cleanup path and ignore cleanup failures because the whole point + of this branch is that the old client/session may already be broken. + */ + if (!client || typeof client.close !== 'function') return; + try { + await client.close(); + } catch (err) { + logger.debug?.('Reolink client close failed', { reason, error: getErrorMessage(err) }); + } +} + +async function resetReolinkClient(reason = 'reset') { + const previous = reolinkClient; + reolinkClient = null; + await closeReolinkClient(previous, reason); +} + +async function createReolinkClient() { /* reolink-nvr-api is published as an ESM-only package. This server is still CommonJS, so a top-level require() fails before the service can even start. @@ -599,7 +680,7 @@ async function ensureReolinkClient() { reolinkModulePromise = import('reolink-nvr-api'); } const { ReolinkClient } = await reolinkModulePromise; - reolinkClient = new ReolinkClient({ + const client = new ReolinkClient({ host: cameraConfig.host, username: cameraConfig.username, password: cameraConfig.password, @@ -607,24 +688,98 @@ async function ensureReolinkClient() { insecure: true, timeout: 10000, }); - await reolinkClient.login(); + await client.login(); + reolinkClient = client; + updateReolinkApiState({ + connected: true, + connecting: false, + retryAt: null, + lastError: null, + lastConnectedAt: Date.now(), + lastEvent: 'connected', + }, 'reolink-api-connected'); return reolinkClient; } +async function ensureReolinkClient() { + if (reolinkClient) return reolinkClient; + updateReolinkApiState({ + connected: false, + connecting: true, + lastEvent: 'connecting', + }, 'reolink-api-connecting'); + return createReolinkClient(); +} + +async function callReolinkApi(command, payload = {}) { + /* + Reolink's HTTP API is stateful in long mode. When the camera restarts or the + SDK session wedges, retrying the same cached client can leave all later + light/IR operations dead until the Node process restarts. This loop treats + any API/login failure as a disposable session, creates a fresh client, and + retries at one fixed cadence until the command succeeds. The caller usually + sits inside serializeVendorState(), so rapid UI toggles stay ordered behind + the reconnecting operation instead of racing multiple login attempts. + */ + while (enabled) { + try { + const client = await ensureReolinkClient(); + const result = await client.api(command, payload); + updateReolinkApiState({ + connected: true, + connecting: false, + retryAt: null, + lastError: null, + lastConnectedAt: Date.now(), + lastEvent: 'api-ok', + }, 'reolink-api-ok'); + return result; + } catch (err) { + const message = getErrorMessage(err); + const retryAt = Date.now() + REOLINK_API_RETRY_MS; + /* + Do not let one bad long-mode session poison the whole service. Clearing + the cached client before the fixed sleep makes the next loop iteration + perform a full login rather than reusing the session that just failed. + */ + await resetReolinkClient(`api-failed:${command}`); + updateReolinkApiState({ + connected: false, + connecting: true, + retryCount: Number(state.reolinkApi?.retryCount || 0) + 1, + retryAt, + lastError: message, + lastEvent: 'retrying', + }, 'reolink-api-retry'); + if (Date.now() >= reolinkApiLogNextAt) { + reolinkApiLogNextAt = Date.now() + 30000; + logger.warn('Reolink API failed; retrying at fixed interval', { + command, + retryMs: REOLINK_API_RETRY_MS, + error: message, + }); + } + await sleep(REOLINK_API_RETRY_MS); + } + } + throw new Error('PTZ camera disabled'); +} + async function refreshVendorState() { if (!enabled) return; - const client = await ensureReolinkClient(); - const [white, ir] = await Promise.all([ - client.api('GetWhiteLed', { channel: 0 }).catch((err) => ({ error: err.message })), - client.api('GetIrLights', { channel: 0 }).catch((err) => ({ error: err.message })), - ]); + /* + Read these sequentially so a failed request can reset and rebuild the SDK + client before the next request starts. Running both in parallel would let + two calls fight over the same broken cached client during reconnect. + */ + const white = await callReolinkApi('GetWhiteLed', { channel: 0 }); + const ir = await callReolinkApi('GetIrLights', { channel: 0 }); state.light = normalizeSpotlightState(white?.WhiteLed || white || null); state.ir = ir?.IrLights || ir || null; } async function refreshSpotlightState() { - const client = await ensureReolinkClient(); - const white = await client.api('GetWhiteLed', { channel: 0 }); + const white = await callReolinkApi('GetWhiteLed', { channel: 0 }); state.light = normalizeSpotlightState(white?.WhiteLed || white || null); emitChange('light'); return state.light; @@ -668,7 +823,17 @@ async function initialize() { try { onvifCam = await connectOnvif(); state.rtspUri = await getStreamUriForProfile(onvifCam); - await refreshVendorState(); + /* + Reolink light/IR state is useful, but it must not block PTZ startup. The + vendor API can be unavailable while ONVIF and RTSP are already healthy; + because callReolinkApi() retries until reconnect, awaiting this refresh + here would keep the publisher and queue disabled until the HTTP API comes + back. Queue it instead so later light/IR operations naturally wait behind + the reconnecting refresh while video startup continues. + */ + serializeVendorState(() => refreshVendorState()).catch((err) => { + logger.warn('initial Reolink state refresh failed', { error: getErrorMessage(err) }); + }); /* Presets are not required for the camera to be usable. Refresh them during startup so connected clients have the list immediately, but keep failures @@ -1011,7 +1176,6 @@ async function removePreset(socket, payload = {}) { async function setSpotlight(socket, payload = {}) { requireOperator(socket); return serializeVendorState(async () => { - const client = await ensureReolinkClient(); let current = state.light ? normalizeSpotlightState(state.light) : null; if (payload.state === undefined && !current) { /* @@ -1047,7 +1211,7 @@ async function setSpotlight(socket, payload = {}) { */ state.light = next; emitChange('light-pending'); - await client.api('SetWhiteLed', { WhiteLed: cameraPayload }); + await callReolinkApi('SetWhiteLed', { WhiteLed: cameraPayload }); scheduleSpotlightVerification(); return state.light; }); @@ -1057,7 +1221,6 @@ async function setIr(socket, payload = {}) { requireOperator(socket); return serializeVendorState(async () => { const nextState = normalizeIrState(payload.state); - const client = await ensureReolinkClient(); /* The camera requires channel inside IrLights. Without it, SetIrLights returns param error (-4), while the optimistic local state makes the UI @@ -1066,7 +1229,7 @@ async function setIr(socket, payload = {}) { */ state.ir = { ...(state.ir || {}), channel: 0, state: nextState }; emitChange('ir-pending'); - await client.api('SetIrLights', { IrLights: { channel: 0, state: nextState } }); + await callReolinkApi('SetIrLights', { IrLights: { channel: 0, state: nextState } }); await refreshVendorState(); emitChange('ir'); return state.ir; @@ -1094,15 +1257,15 @@ async function disableEmittersForIdle() { } return serializeVendorState(async () => { - const client = await ensureReolinkClient(); const lightPayload = { channel: 0, state: spotlightCameraStateForLogicalOn(false) }; const irPayload = { channel: 0, state: normalizeIrState('off') }; - const failures = []; /* Set the public state before the API calls finish so the UI immediately - reflects the idle policy. If a camera call fails, the result still records - that failure and the next vendor refresh can correct the optimistic state. + reflects the idle policy. These API calls intentionally use the same + fixed-interval reconnect loop as user controls, because idle cleanup is + only useful if it survives a camera API session reset instead of giving up + and leaving emitters in an unknown physical state. */ state.light = normalizeSpotlightState({ ...(state.light || {}), @@ -1115,17 +1278,8 @@ async function disableEmittersForIdle() { }; emitChange('idle-emitters-off-pending'); - try { - await client.api('SetWhiteLed', { WhiteLed: lightPayload }); - } catch (err) { - failures.push({ control: 'spotlight', error: err.message }); - } - - try { - await client.api('SetIrLights', { IrLights: irPayload }); - } catch (err) { - failures.push({ control: 'ir', error: err.message }); - } + await callReolinkApi('SetWhiteLed', { WhiteLed: lightPayload }); + await callReolinkApi('SetIrLights', { IrLights: irPayload }); /* Read back once after the writes so stale optimistic state does not linger @@ -1133,17 +1287,13 @@ async function disableEmittersForIdle() { is user-facing and frequently toggled; idle fires rarely, so one ordered refresh keeps the final state simple. */ - try { - await refreshVendorState(); - } catch (err) { - failures.push({ control: 'refresh', error: err.message }); - } + await refreshVendorState(); emitChange('idle-emitters-off'); return { action: 'disablePtzEmitters', - success: failures.length === 0, - failures, + success: true, + failures: [], }; }); }