mirror of
https://github.com/legop3/MultiRoombaRover.git
synced 2026-11-14 10:42:45 -05:00
slop
This commit is contained in:
@@ -68,6 +68,7 @@ require('./src/services/batteryManager');
|
||||
// subscriptions see fully decoded state without becoming an initialization
|
||||
// dependency of either control path.
|
||||
require('./src/services/fleetReportService');
|
||||
require('./src/services/roverLearningService');
|
||||
require('./src/services/replayEngineV2');
|
||||
// Replay delivery is a core service. It must subscribe before the optional
|
||||
// Discord feature so web requests always have a local delivery path.
|
||||
|
||||
@@ -184,6 +184,7 @@ test('service definitions generate public feature paths without a separate regis
|
||||
{ key: 'discord', path: ['discord', 'enabled'] },
|
||||
{ key: 'socials', path: ['socials', 'enabled'] },
|
||||
{ key: 'fleetReports', path: ['fleetReports', 'enabled'] },
|
||||
{ key: 'roverLearning', path: ['roverLearning', 'enabled'] },
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ const barcodeScanner = require('../services/barcodeScannerService/configuration'
|
||||
const commands = require('../services/operatorCommandService/configuration');
|
||||
const discord = require('../services/discordBotService/configuration');
|
||||
const fleetReports = require('../services/fleetReportService/configuration');
|
||||
const roverLearning = require('../services/roverLearningService/configuration');
|
||||
|
||||
/*
|
||||
Object property order is preserved by JSON serialization and JSON Schema
|
||||
@@ -55,6 +56,7 @@ const definitions = [
|
||||
sessionConfiguration.socials,
|
||||
sessionConfiguration.driverAd,
|
||||
fleetReports,
|
||||
roverLearning,
|
||||
];
|
||||
|
||||
const defaultConfig = Object.fromEntries(
|
||||
|
||||
@@ -11,6 +11,7 @@ const logger = require('../../globals/logger').child('commandService');
|
||||
const { isHeadlightBlocked } = require('../../rewards/definitions/darkness');
|
||||
const homeAssistantService = require('../homeAssistantService');
|
||||
const overcurrentProtectionService = require('../overcurrentProtectionService');
|
||||
const { subscribeCommandRecording, observeCommand, recordCommandRequest } = require('./recording');
|
||||
|
||||
// Command observations are intentionally separate from the global event bus.
|
||||
// Drive and motor commands can run at control-loop frequency, and publishing
|
||||
@@ -96,6 +97,7 @@ function issueCommand(roverId, payload) {
|
||||
const normalizedPayload = normalizeOutboundCommandPayload(payload);
|
||||
const message = { ...normalizedPayload, id };
|
||||
record.ws.send(JSON.stringify(message));
|
||||
observeCommand({ phase: 'dispatched', roverId, commandId: id, command: normalizedPayload });
|
||||
pendingCommands.set(id, { roverId, ts: Date.now(), type: normalizedPayload.type });
|
||||
logger.info('Issued command', roverId, normalizedPayload.type, id);
|
||||
return id;
|
||||
@@ -130,6 +132,7 @@ function handleAck(msg) {
|
||||
const pending = pendingCommands.get(msg.id);
|
||||
if (!pending) return;
|
||||
pendingCommands.delete(msg.id);
|
||||
observeCommand({ phase: 'acknowledged', roverId: pending.roverId, commandId: msg.id, acknowledgement: msg });
|
||||
logger.info('Command acknowledged', pending.roverId, pending.type, msg.status);
|
||||
io.emit('commandAck', {
|
||||
roverId: pending.roverId,
|
||||
@@ -255,6 +258,7 @@ module.exports = {
|
||||
getRecentDriveActivity,
|
||||
setDriveCooldown,
|
||||
commandEvents,
|
||||
subscribeCommandRecording,
|
||||
};
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
@@ -392,8 +396,9 @@ io.on('connection', (socket) => {
|
||||
}
|
||||
}
|
||||
|
||||
socket.on('command', handleCommand);
|
||||
socket.on('command:issue', handleCommand);
|
||||
const recordedCommand = (request, reply) => recordCommandRequest(socket, request, reply, handleCommand);
|
||||
socket.on('command', recordedCommand);
|
||||
socket.on('command:issue', recordedCommand);
|
||||
|
||||
socket.on('command:updateAllRovers', (_payload = {}, cb) => {
|
||||
const reply = typeof cb === 'function' ? cb : () => {};
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Optional detailed command observation, separate from payload-free analytics.
|
||||
// Subscribers must only enqueue work; recorder failures cannot fail a command.
|
||||
const { AsyncLocalStorage } = require('node:async_hooks');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
|
||||
const observers = new Set();
|
||||
const requests = new AsyncLocalStorage();
|
||||
|
||||
function subscribeCommandRecording(observer) {
|
||||
observers.add(observer);
|
||||
return () => observers.delete(observer);
|
||||
}
|
||||
|
||||
function observeCommand(event) {
|
||||
if (!observers.size) return;
|
||||
const scope = requests.getStore();
|
||||
const context = scope?.active ? scope.context : null;
|
||||
for (const observer of observers) {
|
||||
try {
|
||||
// Snapshot before safety code or a caller mutates a nested payload. Each
|
||||
// subscriber receives its own copy and cannot change command delivery.
|
||||
observer(structuredClone({
|
||||
ts: Date.now(),
|
||||
monotonicNs: process.hrtime.bigint().toString(),
|
||||
source: context ? 'client' : 'server',
|
||||
...context,
|
||||
...event,
|
||||
}));
|
||||
} catch {
|
||||
// Recording is deliberately best-effort at this boundary. The owning
|
||||
// recorder reports its storage health independently of driver replies.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function recordCommandRequest(socket, request, reply, execute) {
|
||||
if (!observers.size) return execute(request, reply);
|
||||
const context = { requestId: randomUUID(), socketId: socket.id, userId: socket.data?.userId || null };
|
||||
const scope = { active: true, context };
|
||||
return requests.run(scope, () => {
|
||||
try {
|
||||
observeCommand({ phase: 'requested', roverId: request?.roverId, command: request });
|
||||
return execute(request, (result) => {
|
||||
observeCommand({ phase: 'result', roverId: request?.roverId, result });
|
||||
if (typeof reply === 'function') reply(result);
|
||||
});
|
||||
} finally {
|
||||
// The command handler is synchronous. A safety timer created inside it
|
||||
// inherits async context but its later commands are server actions.
|
||||
scope.active = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { subscribeCommandRecording, observeCommand, recordCommandRequest };
|
||||
@@ -0,0 +1,19 @@
|
||||
// Metadata only: loading the configuration must never start the learning runtime.
|
||||
const { strictObject, boolean, integer, number } = require('../../configuration/schemaHelpers');
|
||||
|
||||
module.exports = {
|
||||
key: 'roverLearning',
|
||||
feature: true,
|
||||
defaultValue: {
|
||||
enabled: false,
|
||||
recording: { maxGiB: 20, minimumFreeGiB: 2, sessionSeconds: 300 },
|
||||
},
|
||||
schema: strictObject({
|
||||
enabled: boolean({ description: 'Enable rover learning recording. Disabled starts no recording workers or storage. Training and autonomous control are not implemented yet.' }),
|
||||
recording: strictObject({
|
||||
maxGiB: number({ minimum: 0.1, maximum: 100000, description: 'Rolling recording budget in GiB. Checked every 15 seconds; active files can temporarily exceed it. Published models are never pruned.' }),
|
||||
minimumFreeGiB: number({ minimum: 0.1, maximum: 100000, description: 'Pause recording below this amount of available filesystem space, in GiB.' }),
|
||||
sessionSeconds: integer({ minimum: 30, maximum: 3600, description: 'Rotate video and telemetry sessions after this many seconds so completed sessions can be pruned.' }),
|
||||
}, { description: 'Bounded recording storage. Limits never delete published models.', required: ['maxGiB', 'minimumFreeGiB', 'sessionSeconds'] }),
|
||||
}, { description: 'Optional rover learning service. Currently provides passive recording only.', required: ['enabled', 'recording'] }),
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
// The only resident part while disabled is this configuration lifecycle bridge.
|
||||
const { loadConfig, registerConfigurationHandler } = require('../../configuration');
|
||||
const logger = require('../../globals/logger').child('roverLearning');
|
||||
|
||||
let runtime = null;
|
||||
let transitions = Promise.resolve();
|
||||
let shuttingDown = false;
|
||||
|
||||
function apply(config) {
|
||||
const transition = transitions.then(async () => {
|
||||
if (runtime) {
|
||||
const previous = runtime;
|
||||
runtime = null;
|
||||
await previous.stop();
|
||||
}
|
||||
if (!config?.enabled || shuttingDown) return;
|
||||
const { createRuntime } = require('./runtime');
|
||||
const next = createRuntime({ config, logger });
|
||||
try {
|
||||
await next.start();
|
||||
runtime = next;
|
||||
} catch (error) {
|
||||
await next.stop();
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
transitions = transition.catch(() => undefined);
|
||||
return transition;
|
||||
}
|
||||
|
||||
registerConfigurationHandler('roverLearning', apply);
|
||||
apply(loadConfig().roverLearning).catch((error) => {
|
||||
logger.error('Unable to start rover learning recording', { error: error.message });
|
||||
});
|
||||
|
||||
// Signal handlers are installed by the enabled runtime, not by this bridge.
|
||||
module.exports = {
|
||||
stop: () => { shuttingDown = true; return apply({ enabled: false }); },
|
||||
};
|
||||
@@ -0,0 +1,119 @@
|
||||
// One bounded recording session. Video packets retain receiver wall-clock PTS;
|
||||
// telemetry uses the same server clock, not an assumed browser playback clock.
|
||||
const fs = require('node:fs');
|
||||
const fsp = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const { spawn } = require('node:child_process');
|
||||
|
||||
const MAX_BUFFER_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
async function saveMetadata(directory, metadata) {
|
||||
const filename = path.join(directory, 'session.json');
|
||||
await fsp.writeFile(`${filename}.tmp`, JSON.stringify(metadata, null, 2));
|
||||
await fsp.rename(`${filename}.tmp`, filename);
|
||||
}
|
||||
|
||||
async function createRecording({ root, roverId, snapshot, logger }) {
|
||||
const id = `${Date.now()}-${randomUUID()}`;
|
||||
const directory = path.join(root, id);
|
||||
await fsp.mkdir(directory);
|
||||
const metadata = {
|
||||
version: 1, id, roverId, startedAt: Date.now(), endedAt: null,
|
||||
timing: 'Video PTS uses server receiver Unix seconds. Events use Unix milliseconds and process monotonic nanoseconds. Browser display delay is not measured.',
|
||||
video: { file: 'video.mkv', codec: 'source-copy', timestampUnit: 'seconds', origin: 'unix' },
|
||||
events: 'events.ndjson', droppedEvents: 0,
|
||||
};
|
||||
await saveMetadata(directory, metadata);
|
||||
const output = fs.createWriteStream(path.join(directory, 'events.ndjson'), { flags: 'wx' });
|
||||
let failure = null;
|
||||
let stopping = false;
|
||||
let stopPromise;
|
||||
let child;
|
||||
let childClosed = false;
|
||||
let stderr = '';
|
||||
let videoResult = null;
|
||||
let resolveVideo;
|
||||
const videoClosed = new Promise((resolve) => { resolveVideo = resolve; });
|
||||
output.on('error', (error) => {
|
||||
failure = error.message;
|
||||
// Stop file growth immediately when telemetry is no longer writable.
|
||||
child?.kill('SIGTERM');
|
||||
});
|
||||
|
||||
function append(kind, data, timestamp = {}) {
|
||||
if (stopping || failure) return false;
|
||||
try {
|
||||
const line = JSON.stringify({
|
||||
ts: Date.now(), monotonicNs: process.hrtime.bigint().toString(), ...timestamp, kind, data,
|
||||
}) + '\n';
|
||||
// Never wait for disk from a control/sensor callback. Explicit loss counts
|
||||
// prevent an overloaded recorder from pretending its history is complete.
|
||||
if (output.writableLength + Buffer.byteLength(line) > MAX_BUFFER_BYTES) {
|
||||
metadata.droppedEvents += 1;
|
||||
return true;
|
||||
}
|
||||
output.write(line);
|
||||
} catch (error) {
|
||||
metadata.droppedEvents += 1;
|
||||
failure = error.message;
|
||||
child?.kill('SIGTERM');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
append('snapshot', snapshot);
|
||||
// Read the local published stream, as replay does, without depending on the
|
||||
// replay service or re-encoding camera video. No audio/chat is captured.
|
||||
child = spawn(process.env.FFMPEG_BIN || 'ffmpeg', [
|
||||
'-hide_banner', '-loglevel', 'warning', '-nostdin', '-n',
|
||||
'-rtsp_transport', 'tcp', '-timeout', '10000000',
|
||||
'-use_wallclock_as_timestamps', '1', '-copyts',
|
||||
'-i', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`,
|
||||
'-map', '0:v:0', '-an', '-c:v', 'copy', '-avoid_negative_ts', 'disabled',
|
||||
'-f', 'matroska', path.join(directory, 'video.mkv'),
|
||||
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
||||
child.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString()).slice(-4096); });
|
||||
child.on('error', (error) => { failure = error.message; });
|
||||
child.once('close', (code, signal) => {
|
||||
childClosed = true;
|
||||
videoResult = { code, signal, stderr };
|
||||
if (!stopping) failure ||= `Video recorder exited (${code ?? signal})`;
|
||||
resolveVideo();
|
||||
});
|
||||
|
||||
async function stop(reason) {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
append('sessionEnd', { reason, droppedEvents: metadata.droppedEvents });
|
||||
stopping = true;
|
||||
if (!childClosed) child.kill('SIGTERM');
|
||||
// A stuck RTSP read must not leave a recorder behind after disable.
|
||||
const killTimer = setTimeout(() => { if (!childClosed) child.kill('SIGKILL'); }, 2000);
|
||||
await videoClosed;
|
||||
clearTimeout(killTimer);
|
||||
await new Promise((resolve) => {
|
||||
if (output.closed) return resolve();
|
||||
output.once('close', resolve);
|
||||
output.end();
|
||||
});
|
||||
metadata.endedAt = Date.now();
|
||||
metadata.reason = reason;
|
||||
metadata.error = failure;
|
||||
metadata.video.result = videoResult;
|
||||
await saveMetadata(directory, metadata);
|
||||
if (failure) logger.warn('Recording session incomplete', { roverId, id, error: failure, stderr });
|
||||
})();
|
||||
return stopPromise;
|
||||
}
|
||||
|
||||
return {
|
||||
id, directory, startedAt: metadata.startedAt, append, stop,
|
||||
get failed() { return Boolean(failure); },
|
||||
get stopping() { return stopping; },
|
||||
// Process exit is synchronous; normal disable uses stop() and awaits close.
|
||||
kill: () => { if (!childClosed) child.kill('SIGKILL'); },
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { createRecording, saveMetadata };
|
||||
@@ -0,0 +1,254 @@
|
||||
// Enabled-only recording runtime. No training or driving worker is started here.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { resolveDataPath } = require('../../helpers/dataPaths');
|
||||
const roverManager = require('../roverManager');
|
||||
const io = require('../../globals/io');
|
||||
const { turnEvents } = require('../turnService');
|
||||
const { modeEvents } = require('../modeManager');
|
||||
const { isAdmin, roleEvents } = require('../roleService');
|
||||
const { isDeterred, verificationEvents } = require('../verificationService');
|
||||
const { subscribeCommandRecording, commandEvents } = require('../commandService');
|
||||
const { createRecording } = require('./recording');
|
||||
const { recoverRecordings, enforceRetention } = require('./storage');
|
||||
|
||||
function createRuntime({ config, logger }) {
|
||||
const root = resolveDataPath('rover-learning');
|
||||
const recordingsRoot = path.join(root, 'recordings');
|
||||
const sessions = new Map();
|
||||
const unsubscribers = [];
|
||||
let running = false;
|
||||
let timer;
|
||||
let maintenance = Promise.resolve();
|
||||
let stopPromise;
|
||||
let diskPaused = false;
|
||||
let busy = false;
|
||||
let wakeRequested = false;
|
||||
let pending = [];
|
||||
let pendingBytes = 0;
|
||||
let droppedPending = 0;
|
||||
let eligibleRovers = new Set();
|
||||
|
||||
function hasHumanController(roverId) {
|
||||
const record = roverManager.rovers.get(roverId);
|
||||
if (!record?.ws) return false;
|
||||
// Admin permission alone must not record the whole fleet. Require actual
|
||||
// driver membership, a live browser socket, and current control eligibility.
|
||||
for (const socketId of record.drivers || []) {
|
||||
const socket = io.sockets.sockets.get(socketId);
|
||||
if (socket?.connected && (isAdmin(socket) || !isDeterred(socket))
|
||||
&& roverManager.canDrive(roverId, socket)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function currentEligibleRovers() {
|
||||
return new Set([...roverManager.rovers.keys()].filter(hasHumanController));
|
||||
}
|
||||
|
||||
function refreshEligibility() {
|
||||
const next = currentEligibleRovers();
|
||||
const changed = next.size !== eligibleRovers.size || [...next].some((id) => !eligibleRovers.has(id));
|
||||
eligibleRovers = next;
|
||||
if (!changed) return;
|
||||
// Stop the stream immediately even if disk maintenance is in progress.
|
||||
// The serialized maintenance pass still owns closing/removing the session.
|
||||
for (const [id, session] of sessions) {
|
||||
if (!next.has(id)) session.stop('no-human-controller').catch((error) => {
|
||||
logger.warn('Unable to finalize unattended recording', { roverId: id, error: error.message });
|
||||
});
|
||||
}
|
||||
schedule();
|
||||
}
|
||||
|
||||
function append(kind, event) {
|
||||
if (!running || event?.roverId == null) return;
|
||||
const roverId = String(event.roverId);
|
||||
if (!hasHumanController(roverId)) return;
|
||||
const timestamp = { ts: Date.now(), monotonicNs: process.hrtime.bigint().toString() };
|
||||
if (sessions.get(roverId)?.append(kind, event, timestamp)) return;
|
||||
if (diskPaused || !roverManager.rovers.has(roverId)) return;
|
||||
// Session creation/rotation is asynchronous. Keep a bounded snapshot of
|
||||
// events arriving during that gap, with their original observation times.
|
||||
try {
|
||||
const serialized = JSON.stringify(event);
|
||||
const bytes = Buffer.byteLength(serialized);
|
||||
if (pendingBytes + bytes > 4 * 1024 * 1024) {
|
||||
droppedPending += 1;
|
||||
return;
|
||||
}
|
||||
pending.push({ roverId, kind, serialized, timestamp, bytes });
|
||||
pendingBytes += bytes;
|
||||
} catch { droppedPending += 1; }
|
||||
}
|
||||
|
||||
function listen(emitter, name, handler) {
|
||||
emitter.on(name, handler);
|
||||
unsubscribers.push(() => emitter.off(name, handler));
|
||||
}
|
||||
|
||||
function snapshot(record) {
|
||||
// Do not serialize the live record: it contains sockets and circular state.
|
||||
return {
|
||||
meta: record.meta, sensors: record.lastSensor, hostStats: record.lastHostStats,
|
||||
batteryState: record.batteryState, headlightState: record.headlightState,
|
||||
laserState: record.laserState, drivers: [...(record.drivers || [])],
|
||||
locked: record.locked, privateOpen: record.privateOpen,
|
||||
};
|
||||
}
|
||||
|
||||
async function closeSession(roverId, reason) {
|
||||
const session = sessions.get(roverId);
|
||||
if (!session) return;
|
||||
try {
|
||||
await session.stop(reason);
|
||||
} finally {
|
||||
// Keep closing children reachable by the synchronous process-exit hook.
|
||||
sessions.delete(roverId);
|
||||
}
|
||||
}
|
||||
|
||||
async function maintain() {
|
||||
if (!running) return;
|
||||
pending = pending.filter((event) => hasHumanController(event.roverId));
|
||||
pendingBytes = pending.reduce((sum, event) => sum + event.bytes, 0);
|
||||
for (const [roverId, session] of sessions) {
|
||||
const record = roverManager.rovers.get(roverId);
|
||||
if (!hasHumanController(roverId) || session.stopping || session.failed || Date.now() - session.startedAt >= config.recording.sessionSeconds * 1000) {
|
||||
await closeSession(roverId, !record?.ws ? 'offline' : !hasHumanController(roverId) ? 'no-human-controller' : session.failed ? 'recorder-failed' : 'rotation');
|
||||
}
|
||||
}
|
||||
if (!running) return;
|
||||
const retention = await enforceRetention(
|
||||
recordingsRoot, new Set([...sessions.values()].map((session) => session.id)),
|
||||
config.recording.maxGiB * 1024 ** 3, config.recording.minimumFreeGiB * 1024 ** 3,
|
||||
);
|
||||
if (retention.paused) {
|
||||
if (!diskPaused) logger.warn('Recording paused for disk budget', retention);
|
||||
diskPaused = true;
|
||||
pending = [];
|
||||
pendingBytes = 0;
|
||||
// Close active files to make them eligible for pruning on the next tick.
|
||||
await Promise.all([...sessions.keys()].map((id) => closeSession(id, 'disk-budget')));
|
||||
return;
|
||||
}
|
||||
if (diskPaused) logger.info('Recording resumed after disk budget recovery');
|
||||
diskPaused = false;
|
||||
for (const [key, record] of roverManager.rovers) {
|
||||
if (!running) break;
|
||||
const roverId = String(key);
|
||||
if (!hasHumanController(roverId) || sessions.has(roverId)) continue;
|
||||
const session = await createRecording({ root: recordingsRoot, roverId, snapshot: snapshot(record), logger });
|
||||
sessions.set(roverId, session);
|
||||
// Filesystem awaits can span a disconnect or turn change.
|
||||
if (!running || !hasHumanController(roverId)) {
|
||||
await closeSession(roverId, running ? 'no-human-controller' : 'disabled');
|
||||
continue;
|
||||
}
|
||||
if (droppedPending) {
|
||||
session.append('captureGap', { droppedEventsAcrossRovers: droppedPending });
|
||||
droppedPending = 0;
|
||||
}
|
||||
pending = pending.filter((event) => {
|
||||
if (event.roverId !== roverId) return true;
|
||||
session.append(event.kind, JSON.parse(event.serialized), event.timestamp);
|
||||
pendingBytes -= event.bytes;
|
||||
return false;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
if (!running) return;
|
||||
if (busy) { wakeRequested = true; return; }
|
||||
clearTimeout(timer);
|
||||
busy = true;
|
||||
// The next tick is scheduled after completion: disk slowness cannot build
|
||||
// an unbounded backlog of retention tasks or overlapping recorder starts.
|
||||
maintenance = maintain().catch(async (error) => {
|
||||
logger.error('Recording maintenance failed; closing recorders', { error: error.message });
|
||||
await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'storage-error')));
|
||||
}).finally(() => {
|
||||
busy = false;
|
||||
if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
|
||||
wakeRequested = false;
|
||||
});
|
||||
}
|
||||
|
||||
function killChildren() { for (const session of sessions.values()) session.kill(); }
|
||||
function onSignal() {
|
||||
// Other server services can exit the process before asynchronous cleanup
|
||||
// finishes. Signal children immediately, and keep an exit fallback too.
|
||||
for (const session of sessions.values()) session.stop('server-shutdown').catch(() => undefined);
|
||||
stop().catch((error) => logger.warn('Recording shutdown failed', { error: error.message }));
|
||||
}
|
||||
|
||||
async function start() {
|
||||
await fs.mkdir(recordingsRoot, { recursive: true });
|
||||
await recoverRecordings(recordingsRoot);
|
||||
running = true;
|
||||
eligibleRovers = currentEligibleRovers();
|
||||
unsubscribers.push(subscribeCommandRecording((event) => append('command', event)));
|
||||
// ACK analytics also provide latency/type context; detailed acknowledgements
|
||||
// arrive through the recording hook and share the command ID.
|
||||
listen(commandEvents, 'observation', (event) => append('commandOutcome', event));
|
||||
listen(roverManager.managerEvents, 'sensor', (event) => append('sensor', {
|
||||
...event,
|
||||
// The manager stores the exact frame before emitting this synchronous
|
||||
// event, so raw capture needs no extra hook in rover sensor processing.
|
||||
frame: roverManager.rovers.get(event.roverId)?.lastSensor?.raw,
|
||||
}));
|
||||
for (const kind of ['hostStats', 'driver', 'switch', 'lock', 'private', 'privateSafety', 'help', 'dockGuard']) {
|
||||
listen(roverManager.managerEvents, kind, (event) => {
|
||||
append(kind, event);
|
||||
if (['driver', 'switch', 'lock', 'private', 'privateSafety'].includes(kind)) refreshEligibility();
|
||||
});
|
||||
}
|
||||
for (const kind of ['activeDriver', 'queue']) {
|
||||
listen(turnEvents, kind, (event) => { append(`turn:${kind}`, event); refreshEligibility(); });
|
||||
}
|
||||
listen(modeEvents, 'change', refreshEligibility);
|
||||
listen(roleEvents, 'change', refreshEligibility);
|
||||
listen(verificationEvents, 'change', refreshEligibility);
|
||||
listen(roverManager.managerEvents, 'rover', (event) => {
|
||||
append('rover', { roverId: event.roverId, action: event.action, snapshot: event.record ? snapshot(event.record) : null });
|
||||
if (event.action === 'upsert' || event.action === 'removed') refreshEligibility();
|
||||
});
|
||||
process.on('SIGTERM', onSignal);
|
||||
process.on('SIGINT', onSignal);
|
||||
process.on('exit', killChildren);
|
||||
busy = true;
|
||||
maintenance = maintain();
|
||||
try {
|
||||
await maintenance;
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
|
||||
wakeRequested = false;
|
||||
logger.info('Rover learning recording enabled', { root });
|
||||
}
|
||||
|
||||
async function stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
running = false;
|
||||
clearTimeout(timer);
|
||||
for (const unsubscribe of unsubscribers.splice(0)) unsubscribe();
|
||||
stopPromise = (async () => {
|
||||
await maintenance.catch(() => undefined);
|
||||
const results = await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'disabled')));
|
||||
process.off('SIGTERM', onSignal);
|
||||
process.off('SIGINT', onSignal);
|
||||
process.off('exit', killChildren);
|
||||
pending = [];
|
||||
pendingBytes = 0;
|
||||
const failed = results.find((result) => result.status === 'rejected');
|
||||
if (failed) throw failed.reason;
|
||||
})();
|
||||
return stopPromise;
|
||||
}
|
||||
|
||||
return { start, stop };
|
||||
}
|
||||
|
||||
module.exports = { createRuntime };
|
||||
@@ -0,0 +1,59 @@
|
||||
// Recording retention only. Published models and training state are never visited.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { saveMetadata } = require('./recording');
|
||||
|
||||
async function listRecordings(root) {
|
||||
const recordings = [];
|
||||
for (const entry of await fs.readdir(root, { withFileTypes: true })) {
|
||||
// Directory names are generated, never rover IDs or user-supplied paths.
|
||||
if (!entry.isDirectory() || !/^\d+-[0-9a-f-]{36}$/.test(entry.name)) continue;
|
||||
const directory = path.join(root, entry.name);
|
||||
let bytes = 0;
|
||||
for (const file of await fs.readdir(directory, { withFileTypes: true })) {
|
||||
if (!file.isFile()) continue;
|
||||
bytes += (await fs.stat(path.join(directory, file.name))).size;
|
||||
}
|
||||
recordings.push({ id: entry.name, directory, bytes, startedAt: Number(entry.name.split('-')[0]) });
|
||||
}
|
||||
return recordings.sort((a, b) => a.startedAt - b.startedAt);
|
||||
}
|
||||
|
||||
async function recoverRecordings(root) {
|
||||
for (const recording of await listRecordings(root)) {
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8'));
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
|
||||
// An interrupted initial metadata write is retained as an unusable session
|
||||
// until ordinary retention removes it, rather than blocking all startup.
|
||||
metadata = { version: 1, id: recording.id, startedAt: recording.startedAt };
|
||||
}
|
||||
if (metadata.endedAt != null) continue;
|
||||
await saveMetadata(recording.directory, {
|
||||
...metadata, endedAt: Date.now(), reason: 'interrupted',
|
||||
error: 'Previous process did not close this session; final timestamps and video may be incomplete.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function enforceRetention(root, activeIds, maxBytes, minimumFreeBytes) {
|
||||
const recordings = await listRecordings(root);
|
||||
let bytes = recordings.reduce((total, recording) => total + recording.bytes, 0);
|
||||
const stat = await fs.statfs(root);
|
||||
let free = stat.bavail * stat.bsize;
|
||||
for (const recording of recordings) {
|
||||
if (bytes < maxBytes && free >= minimumFreeBytes) break;
|
||||
if (activeIds.has(recording.id)) continue;
|
||||
await fs.rm(recording.directory, { recursive: true });
|
||||
bytes -= recording.bytes;
|
||||
// Re-read actual free space; sparse/compressed files and other services can
|
||||
// make a deleted file's logical size a misleading estimate of freed space.
|
||||
const current = await fs.statfs(root);
|
||||
free = current.bavail * current.bsize;
|
||||
}
|
||||
return { bytes, free, paused: bytes >= maxBytes || free < minimumFreeBytes };
|
||||
}
|
||||
|
||||
module.exports = { recoverRecordings, enforceRetention };
|
||||
Reference in New Issue
Block a user