slop

server/src/services/commandService/index.js

@@ -11,6 +11,7 @@ const logger = require('../../globals/logger').child('commandService');
 const { isHeadlightBlocked } = require('../../rewards/definitions/darkness');
 const homeAssistantService = require('../homeAssistantService');
 const overcurrentProtectionService = require('../overcurrentProtectionService');
+const { subscribeCommandRecording, observeCommand, recordCommandRequest } = require('./recording');
 
 // Command observations are intentionally separate from the global event bus.
 // Drive and motor commands can run at control-loop frequency, and publishing
@@ -96,6 +97,7 @@ function issueCommand(roverId, payload) {
   const normalizedPayload = normalizeOutboundCommandPayload(payload);
   const message = { ...normalizedPayload, id };
   record.ws.send(JSON.stringify(message));
+  observeCommand({ phase: 'dispatched', roverId, commandId: id, command: normalizedPayload });
   pendingCommands.set(id, { roverId, ts: Date.now(), type: normalizedPayload.type });
   logger.info('Issued command', roverId, normalizedPayload.type, id);
   return id;
@@ -130,6 +132,7 @@ function handleAck(msg) {
   const pending = pendingCommands.get(msg.id);
   if (!pending) return;
   pendingCommands.delete(msg.id);
+  observeCommand({ phase: 'acknowledged', roverId: pending.roverId, commandId: msg.id, acknowledgement: msg });
   logger.info('Command acknowledged', pending.roverId, pending.type, msg.status);
   io.emit('commandAck', {
     roverId: pending.roverId,
@@ -255,6 +258,7 @@ module.exports = {
   getRecentDriveActivity,
   setDriveCooldown,
   commandEvents,
+  subscribeCommandRecording,
 };
 
 io.on('connection', (socket) => {
@@ -392,8 +396,9 @@ io.on('connection', (socket) => {
     }
   }
 
-  socket.on('command', handleCommand);
-  socket.on('command:issue', handleCommand);
+  const recordedCommand = (request, reply) => recordCommandRequest(socket, request, reply, handleCommand);
+  socket.on('command', recordedCommand);
+  socket.on('command:issue', recordedCommand);
 
   socket.on('command:updateAllRovers', (_payload = {}, cb) => {
     const reply = typeof cb === 'function' ? cb : () => {};

server/src/services/commandService/recording.js

@@ -0,0 +1,55 @@
+// Optional detailed command observation, separate from payload-free analytics.
+// Subscribers must only enqueue work; recorder failures cannot fail a command.
+const { AsyncLocalStorage } = require('node:async_hooks');
+const { randomUUID } = require('node:crypto');
+
+const observers = new Set();
+const requests = new AsyncLocalStorage();
+
+function subscribeCommandRecording(observer) {
+  observers.add(observer);
+  return () => observers.delete(observer);
+}
+
+function observeCommand(event) {
+  if (!observers.size) return;
+  const scope = requests.getStore();
+  const context = scope?.active ? scope.context : null;
+  for (const observer of observers) {
+    try {
+      // Snapshot before safety code or a caller mutates a nested payload. Each
+      // subscriber receives its own copy and cannot change command delivery.
+      observer(structuredClone({
+        ts: Date.now(),
+        monotonicNs: process.hrtime.bigint().toString(),
+        source: context ? 'client' : 'server',
+        ...context,
+        ...event,
+      }));
+    } catch {
+      // Recording is deliberately best-effort at this boundary. The owning
+      // recorder reports its storage health independently of driver replies.
+    }
+  }
+}
+
+function recordCommandRequest(socket, request, reply, execute) {
+  if (!observers.size) return execute(request, reply);
+  const context = { requestId: randomUUID(), socketId: socket.id, userId: socket.data?.userId || null };
+  const scope = { active: true, context };
+  return requests.run(scope, () => {
+    try {
+      observeCommand({ phase: 'requested', roverId: request?.roverId, command: request });
+      return execute(request, (result) => {
+        observeCommand({ phase: 'result', roverId: request?.roverId, result });
+        if (typeof reply === 'function') reply(result);
+      });
+    } finally {
+      // The command handler is synchronous. A safety timer created inside it
+      // inherits async context but its later commands are server actions.
+      scope.active = false;
+    }
+  });
+}
+
+module.exports = { subscribeCommandRecording, observeCommand, recordCommandRequest };

server/src/services/roverLearningService/configuration.js

@@ -0,0 +1,19 @@
+// Metadata only: loading the configuration must never start the learning runtime.
+const { strictObject, boolean, integer, number } = require('../../configuration/schemaHelpers');
+
+module.exports = {
+  key: 'roverLearning',
+  feature: true,
+  defaultValue: {
+    enabled: false,
+    recording: { maxGiB: 20, minimumFreeGiB: 2, sessionSeconds: 300 },
+  },
+  schema: strictObject({
+    enabled: boolean({ description: 'Enable rover learning recording. Disabled starts no recording workers or storage. Training and autonomous control are not implemented yet.' }),
+    recording: strictObject({
+      maxGiB: number({ minimum: 0.1, maximum: 100000, description: 'Rolling recording budget in GiB. Checked every 15 seconds; active files can temporarily exceed it. Published models are never pruned.' }),
+      minimumFreeGiB: number({ minimum: 0.1, maximum: 100000, description: 'Pause recording below this amount of available filesystem space, in GiB.' }),
+      sessionSeconds: integer({ minimum: 30, maximum: 3600, description: 'Rotate video and telemetry sessions after this many seconds so completed sessions can be pruned.' }),
+    }, { description: 'Bounded recording storage. Limits never delete published models.', required: ['maxGiB', 'minimumFreeGiB', 'sessionSeconds'] }),
+  }, { description: 'Optional rover learning service. Currently provides passive recording only.', required: ['enabled', 'recording'] }),
+};

server/src/services/roverLearningService/index.js

@@ -0,0 +1,39 @@
+// The only resident part while disabled is this configuration lifecycle bridge.
+const { loadConfig, registerConfigurationHandler } = require('../../configuration');
+const logger = require('../../globals/logger').child('roverLearning');
+
+let runtime = null;
+let transitions = Promise.resolve();
+let shuttingDown = false;
+
+function apply(config) {
+  const transition = transitions.then(async () => {
+    if (runtime) {
+      const previous = runtime;
+      runtime = null;
+      await previous.stop();
+    }
+    if (!config?.enabled || shuttingDown) return;
+    const { createRuntime } = require('./runtime');
+    const next = createRuntime({ config, logger });
+    try {
+      await next.start();
+      runtime = next;
+    } catch (error) {
+      await next.stop();
+      throw error;
+    }
+  });
+  transitions = transition.catch(() => undefined);
+  return transition;
+}
+
+registerConfigurationHandler('roverLearning', apply);
+apply(loadConfig().roverLearning).catch((error) => {
+  logger.error('Unable to start rover learning recording', { error: error.message });
+});
+
+// Signal handlers are installed by the enabled runtime, not by this bridge.
+module.exports = {
+  stop: () => { shuttingDown = true; return apply({ enabled: false }); },
+};

server/src/services/roverLearningService/recording.js

@@ -0,0 +1,119 @@
+// One bounded recording session. Video packets retain receiver wall-clock PTS;
+// telemetry uses the same server clock, not an assumed browser playback clock.
+const fs = require('node:fs');
+const fsp = require('node:fs/promises');
+const path = require('node:path');
+const { randomUUID } = require('node:crypto');
+const { spawn } = require('node:child_process');
+
+const MAX_BUFFER_BYTES = 2 * 1024 * 1024;
+
+async function saveMetadata(directory, metadata) {
+  const filename = path.join(directory, 'session.json');
+  await fsp.writeFile(`${filename}.tmp`, JSON.stringify(metadata, null, 2));
+  await fsp.rename(`${filename}.tmp`, filename);
+}
+
+async function createRecording({ root, roverId, snapshot, logger }) {
+  const id = `${Date.now()}-${randomUUID()}`;
+  const directory = path.join(root, id);
+  await fsp.mkdir(directory);
+  const metadata = {
+    version: 1, id, roverId, startedAt: Date.now(), endedAt: null,
+    timing: 'Video PTS uses server receiver Unix seconds. Events use Unix milliseconds and process monotonic nanoseconds. Browser display delay is not measured.',
+    video: { file: 'video.mkv', codec: 'source-copy', timestampUnit: 'seconds', origin: 'unix' },
+    events: 'events.ndjson', droppedEvents: 0,
+  };
+  await saveMetadata(directory, metadata);
+  const output = fs.createWriteStream(path.join(directory, 'events.ndjson'), { flags: 'wx' });
+  let failure = null;
+  let stopping = false;
+  let stopPromise;
+  let child;
+  let childClosed = false;
+  let stderr = '';
+  let videoResult = null;
+  let resolveVideo;
+  const videoClosed = new Promise((resolve) => { resolveVideo = resolve; });
+  output.on('error', (error) => {
+    failure = error.message;
+    // Stop file growth immediately when telemetry is no longer writable.
+    child?.kill('SIGTERM');
+  });
+
+  function append(kind, data, timestamp = {}) {
+    if (stopping || failure) return false;
+    try {
+      const line = JSON.stringify({
+        ts: Date.now(), monotonicNs: process.hrtime.bigint().toString(), ...timestamp, kind, data,
+      }) + '\n';
+      // Never wait for disk from a control/sensor callback. Explicit loss counts
+      // prevent an overloaded recorder from pretending its history is complete.
+      if (output.writableLength + Buffer.byteLength(line) > MAX_BUFFER_BYTES) {
+        metadata.droppedEvents += 1;
+        return true;
+      }
+      output.write(line);
+    } catch (error) {
+      metadata.droppedEvents += 1;
+      failure = error.message;
+      child?.kill('SIGTERM');
+    }
+    return true;
+  }
+
+  append('snapshot', snapshot);
+  // Read the local published stream, as replay does, without depending on the
+  // replay service or re-encoding camera video. No audio/chat is captured.
+  child = spawn(process.env.FFMPEG_BIN || 'ffmpeg', [
+    '-hide_banner', '-loglevel', 'warning', '-nostdin', '-n',
+    '-rtsp_transport', 'tcp', '-timeout', '10000000',
+    '-use_wallclock_as_timestamps', '1', '-copyts',
+    '-i', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`,
+    '-map', '0:v:0', '-an', '-c:v', 'copy', '-avoid_negative_ts', 'disabled',
+    '-f', 'matroska', path.join(directory, 'video.mkv'),
+  ], { stdio: ['ignore', 'ignore', 'pipe'] });
+  child.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString()).slice(-4096); });
+  child.on('error', (error) => { failure = error.message; });
+  child.once('close', (code, signal) => {
+    childClosed = true;
+    videoResult = { code, signal, stderr };
+    if (!stopping) failure ||= `Video recorder exited (${code ?? signal})`;
+    resolveVideo();
+  });
+
+  async function stop(reason) {
+    if (stopPromise) return stopPromise;
+    stopPromise = (async () => {
+      append('sessionEnd', { reason, droppedEvents: metadata.droppedEvents });
+      stopping = true;
+      if (!childClosed) child.kill('SIGTERM');
+      // A stuck RTSP read must not leave a recorder behind after disable.
+      const killTimer = setTimeout(() => { if (!childClosed) child.kill('SIGKILL'); }, 2000);
+      await videoClosed;
+      clearTimeout(killTimer);
+      await new Promise((resolve) => {
+        if (output.closed) return resolve();
+        output.once('close', resolve);
+        output.end();
+      });
+      metadata.endedAt = Date.now();
+      metadata.reason = reason;
+      metadata.error = failure;
+      metadata.video.result = videoResult;
+      await saveMetadata(directory, metadata);
+      if (failure) logger.warn('Recording session incomplete', { roverId, id, error: failure, stderr });
+    })();
+    return stopPromise;
+  }
+
+  return {
+    id, directory, startedAt: metadata.startedAt, append, stop,
+    get failed() { return Boolean(failure); },
+    get stopping() { return stopping; },
+    // Process exit is synchronous; normal disable uses stop() and awaits close.
+    kill: () => { if (!childClosed) child.kill('SIGKILL'); },
+  };
+}
+
+module.exports = { createRecording, saveMetadata };

server/src/services/roverLearningService/runtime.js

@@ -0,0 +1,254 @@
+// Enabled-only recording runtime. No training or driving worker is started here.
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { resolveDataPath } = require('../../helpers/dataPaths');
+const roverManager = require('../roverManager');
+const io = require('../../globals/io');
+const { turnEvents } = require('../turnService');
+const { modeEvents } = require('../modeManager');
+const { isAdmin, roleEvents } = require('../roleService');
+const { isDeterred, verificationEvents } = require('../verificationService');
+const { subscribeCommandRecording, commandEvents } = require('../commandService');
+const { createRecording } = require('./recording');
+const { recoverRecordings, enforceRetention } = require('./storage');
+
+function createRuntime({ config, logger }) {
+  const root = resolveDataPath('rover-learning');
+  const recordingsRoot = path.join(root, 'recordings');
+  const sessions = new Map();
+  const unsubscribers = [];
+  let running = false;
+  let timer;
+  let maintenance = Promise.resolve();
+  let stopPromise;
+  let diskPaused = false;
+  let busy = false;
+  let wakeRequested = false;
+  let pending = [];
+  let pendingBytes = 0;
+  let droppedPending = 0;
+  let eligibleRovers = new Set();
+
+  function hasHumanController(roverId) {
+    const record = roverManager.rovers.get(roverId);
+    if (!record?.ws) return false;
+    // Admin permission alone must not record the whole fleet. Require actual
+    // driver membership, a live browser socket, and current control eligibility.
+    for (const socketId of record.drivers || []) {
+      const socket = io.sockets.sockets.get(socketId);
+      if (socket?.connected && (isAdmin(socket) || !isDeterred(socket))
+          && roverManager.canDrive(roverId, socket)) return true;
+    }
+    return false;
+  }
+
+  function currentEligibleRovers() {
+    return new Set([...roverManager.rovers.keys()].filter(hasHumanController));
+  }
+
+  function refreshEligibility() {
+    const next = currentEligibleRovers();
+    const changed = next.size !== eligibleRovers.size || [...next].some((id) => !eligibleRovers.has(id));
+    eligibleRovers = next;
+    if (!changed) return;
+    // Stop the stream immediately even if disk maintenance is in progress.
+    // The serialized maintenance pass still owns closing/removing the session.
+    for (const [id, session] of sessions) {
+      if (!next.has(id)) session.stop('no-human-controller').catch((error) => {
+        logger.warn('Unable to finalize unattended recording', { roverId: id, error: error.message });
+      });
+    }
+    schedule();
+  }
+
+  function append(kind, event) {
+    if (!running || event?.roverId == null) return;
+    const roverId = String(event.roverId);
+    if (!hasHumanController(roverId)) return;
+    const timestamp = { ts: Date.now(), monotonicNs: process.hrtime.bigint().toString() };
+    if (sessions.get(roverId)?.append(kind, event, timestamp)) return;
+    if (diskPaused || !roverManager.rovers.has(roverId)) return;
+    // Session creation/rotation is asynchronous. Keep a bounded snapshot of
+    // events arriving during that gap, with their original observation times.
+    try {
+      const serialized = JSON.stringify(event);
+      const bytes = Buffer.byteLength(serialized);
+      if (pendingBytes + bytes > 4 * 1024 * 1024) {
+        droppedPending += 1;
+        return;
+      }
+      pending.push({ roverId, kind, serialized, timestamp, bytes });
+      pendingBytes += bytes;
+    } catch { droppedPending += 1; }
+  }
+
+  function listen(emitter, name, handler) {
+    emitter.on(name, handler);
+    unsubscribers.push(() => emitter.off(name, handler));
+  }
+
+  function snapshot(record) {
+    // Do not serialize the live record: it contains sockets and circular state.
+    return {
+      meta: record.meta, sensors: record.lastSensor, hostStats: record.lastHostStats,
+      batteryState: record.batteryState, headlightState: record.headlightState,
+      laserState: record.laserState, drivers: [...(record.drivers || [])],
+      locked: record.locked, privateOpen: record.privateOpen,
+    };
+  }
+
+  async function closeSession(roverId, reason) {
+    const session = sessions.get(roverId);
+    if (!session) return;
+    try {
+      await session.stop(reason);
+    } finally {
+      // Keep closing children reachable by the synchronous process-exit hook.
+      sessions.delete(roverId);
+    }
+  }
+
+  async function maintain() {
+    if (!running) return;
+    pending = pending.filter((event) => hasHumanController(event.roverId));
+    pendingBytes = pending.reduce((sum, event) => sum + event.bytes, 0);
+    for (const [roverId, session] of sessions) {
+      const record = roverManager.rovers.get(roverId);
+      if (!hasHumanController(roverId) || session.stopping || session.failed || Date.now() - session.startedAt >= config.recording.sessionSeconds * 1000) {
+        await closeSession(roverId, !record?.ws ? 'offline' : !hasHumanController(roverId) ? 'no-human-controller' : session.failed ? 'recorder-failed' : 'rotation');
+      }
+    }
+    if (!running) return;
+    const retention = await enforceRetention(
+      recordingsRoot, new Set([...sessions.values()].map((session) => session.id)),
+      config.recording.maxGiB * 1024 ** 3, config.recording.minimumFreeGiB * 1024 ** 3,
+    );
+    if (retention.paused) {
+      if (!diskPaused) logger.warn('Recording paused for disk budget', retention);
+      diskPaused = true;
+      pending = [];
+      pendingBytes = 0;
+      // Close active files to make them eligible for pruning on the next tick.
+      await Promise.all([...sessions.keys()].map((id) => closeSession(id, 'disk-budget')));
+      return;
+    }
+    if (diskPaused) logger.info('Recording resumed after disk budget recovery');
+    diskPaused = false;
+    for (const [key, record] of roverManager.rovers) {
+      if (!running) break;
+      const roverId = String(key);
+      if (!hasHumanController(roverId) || sessions.has(roverId)) continue;
+      const session = await createRecording({ root: recordingsRoot, roverId, snapshot: snapshot(record), logger });
+      sessions.set(roverId, session);
+      // Filesystem awaits can span a disconnect or turn change.
+      if (!running || !hasHumanController(roverId)) {
+        await closeSession(roverId, running ? 'no-human-controller' : 'disabled');
+        continue;
+      }
+      if (droppedPending) {
+        session.append('captureGap', { droppedEventsAcrossRovers: droppedPending });
+        droppedPending = 0;
+      }
+      pending = pending.filter((event) => {
+        if (event.roverId !== roverId) return true;
+        session.append(event.kind, JSON.parse(event.serialized), event.timestamp);
+        pendingBytes -= event.bytes;
+        return false;
+      });
+    }
+  }
+
+  function schedule() {
+    if (!running) return;
+    if (busy) { wakeRequested = true; return; }
+    clearTimeout(timer);
+    busy = true;
+    // The next tick is scheduled after completion: disk slowness cannot build
+    // an unbounded backlog of retention tasks or overlapping recorder starts.
+    maintenance = maintain().catch(async (error) => {
+      logger.error('Recording maintenance failed; closing recorders', { error: error.message });
+      await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'storage-error')));
+    }).finally(() => {
+      busy = false;
+      if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
+      wakeRequested = false;
+    });
+  }
+
+  function killChildren() { for (const session of sessions.values()) session.kill(); }
+  function onSignal() {
+    // Other server services can exit the process before asynchronous cleanup
+    // finishes. Signal children immediately, and keep an exit fallback too.
+    for (const session of sessions.values()) session.stop('server-shutdown').catch(() => undefined);
+    stop().catch((error) => logger.warn('Recording shutdown failed', { error: error.message }));
+  }
+
+  async function start() {
+    await fs.mkdir(recordingsRoot, { recursive: true });
+    await recoverRecordings(recordingsRoot);
+    running = true;
+    eligibleRovers = currentEligibleRovers();
+    unsubscribers.push(subscribeCommandRecording((event) => append('command', event)));
+    // ACK analytics also provide latency/type context; detailed acknowledgements
+    // arrive through the recording hook and share the command ID.
+    listen(commandEvents, 'observation', (event) => append('commandOutcome', event));
+    listen(roverManager.managerEvents, 'sensor', (event) => append('sensor', {
+      ...event,
+      // The manager stores the exact frame before emitting this synchronous
+      // event, so raw capture needs no extra hook in rover sensor processing.
+      frame: roverManager.rovers.get(event.roverId)?.lastSensor?.raw,
+    }));
+    for (const kind of ['hostStats', 'driver', 'switch', 'lock', 'private', 'privateSafety', 'help', 'dockGuard']) {
+      listen(roverManager.managerEvents, kind, (event) => {
+        append(kind, event);
+        if (['driver', 'switch', 'lock', 'private', 'privateSafety'].includes(kind)) refreshEligibility();
+      });
+    }
+    for (const kind of ['activeDriver', 'queue']) {
+      listen(turnEvents, kind, (event) => { append(`turn:${kind}`, event); refreshEligibility(); });
+    }
+    listen(modeEvents, 'change', refreshEligibility);
+    listen(roleEvents, 'change', refreshEligibility);
+    listen(verificationEvents, 'change', refreshEligibility);
+    listen(roverManager.managerEvents, 'rover', (event) => {
+      append('rover', { roverId: event.roverId, action: event.action, snapshot: event.record ? snapshot(event.record) : null });
+      if (event.action === 'upsert' || event.action === 'removed') refreshEligibility();
+    });
+    process.on('SIGTERM', onSignal);
+    process.on('SIGINT', onSignal);
+    process.on('exit', killChildren);
+    busy = true;
+    maintenance = maintain();
+    try {
+      await maintenance;
+    } finally {
+      busy = false;
+    }
+    if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
+    wakeRequested = false;
+    logger.info('Rover learning recording enabled', { root });
+  }
+
+  async function stop() {
+    if (stopPromise) return stopPromise;
+    running = false;
+    clearTimeout(timer);
+    for (const unsubscribe of unsubscribers.splice(0)) unsubscribe();
+    stopPromise = (async () => {
+      await maintenance.catch(() => undefined);
+      const results = await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'disabled')));
+      process.off('SIGTERM', onSignal);
+      process.off('SIGINT', onSignal);
+      process.off('exit', killChildren);
+      pending = [];
+      pendingBytes = 0;
+      const failed = results.find((result) => result.status === 'rejected');
+      if (failed) throw failed.reason;
+    })();
+    return stopPromise;
+  }
+
+  return { start, stop };
+}
+
+module.exports = { createRuntime };

server/src/services/roverLearningService/storage.js

@@ -0,0 +1,59 @@
+// Recording retention only. Published models and training state are never visited.
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { saveMetadata } = require('./recording');
+
+async function listRecordings(root) {
+  const recordings = [];
+  for (const entry of await fs.readdir(root, { withFileTypes: true })) {
+    // Directory names are generated, never rover IDs or user-supplied paths.
+    if (!entry.isDirectory() || !/^\d+-[0-9a-f-]{36}$/.test(entry.name)) continue;
+    const directory = path.join(root, entry.name);
+    let bytes = 0;
+    for (const file of await fs.readdir(directory, { withFileTypes: true })) {
+      if (!file.isFile()) continue;
+      bytes += (await fs.stat(path.join(directory, file.name))).size;
+    }
+    recordings.push({ id: entry.name, directory, bytes, startedAt: Number(entry.name.split('-')[0]) });
+  }
+  return recordings.sort((a, b) => a.startedAt - b.startedAt);
+}
+
+async function recoverRecordings(root) {
+  for (const recording of await listRecordings(root)) {
+    let metadata;
+    try {
+      metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8'));
+    } catch (error) {
+      if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
+      // An interrupted initial metadata write is retained as an unusable session
+      // until ordinary retention removes it, rather than blocking all startup.
+      metadata = { version: 1, id: recording.id, startedAt: recording.startedAt };
+    }
+    if (metadata.endedAt != null) continue;
+    await saveMetadata(recording.directory, {
+      ...metadata, endedAt: Date.now(), reason: 'interrupted',
+      error: 'Previous process did not close this session; final timestamps and video may be incomplete.',
+    });
+  }
+}
+
+async function enforceRetention(root, activeIds, maxBytes, minimumFreeBytes) {
+  const recordings = await listRecordings(root);
+  let bytes = recordings.reduce((total, recording) => total + recording.bytes, 0);
+  const stat = await fs.statfs(root);
+  let free = stat.bavail * stat.bsize;
+  for (const recording of recordings) {
+    if (bytes < maxBytes && free >= minimumFreeBytes) break;
+    if (activeIds.has(recording.id)) continue;
+    await fs.rm(recording.directory, { recursive: true });
+    bytes -= recording.bytes;
+    // Re-read actual free space; sparse/compressed files and other services can
+    // make a deleted file's logical size a misleading estimate of freed space.
+    const current = await fs.statfs(root);
+    free = current.bavail * current.bsize;
+  }
+  return { bytes, free, paused: bytes >= maxBytes || free < minimumFreeBytes };
+}
+
+module.exports = { recoverRecordings, enforceRetention };