micrationslop

Feat rover learner model

Dockerfile

@@ -97,6 +97,15 @@ RUN curl --fail --location --retry 3 \
     && find /output/opt/roverd/googletts -type d -exec chmod 0755 {} + \
     && find /output/opt/roverd/googletts -type f -exec chmod 0644 {} +
 
+FROM architecture-check AS learning-dependencies
+# CPU-only learning dependencies are built once. Disabled installations never
+# launch Python or install packages as a side effect of starting Node.
+RUN dnf install -y --setopt=install_weak_deps=False python3 python3-pip \
+    && dnf clean all
+COPY server/src/services/roverLearningService/workers/requirements.txt /tmp/learning-requirements.txt
+RUN python3 -m venv /opt/rover-learning \
+    && /opt/rover-learning/bin/pip install --no-cache-dir -r /tmp/learning-requirements.txt
+
 FROM fedora:${FEDORA_VERSION} AS runtime
 ARG FEDORA_VERSION
 ARG TARGETARCH
@@ -148,6 +157,7 @@ COPY --from=webui-build /build/server/public ./public
 COPY --from=native-workers /build/kinect/kinect_worker ./src/services/kinectService/native/kinect_worker
 COPY --from=native-workers /build/balance-board/balance_board_worker ./src/services/balanceBoardService/native/balance_board_worker
 COPY --from=packaged-tools /output/ /
+COPY --from=learning-dependencies /opt/rover-learning /opt/rover-learning
 COPY --chmod=0755 server/bin/chromegtts-wav.py /usr/local/bin/chromegtts-wav
 COPY --chmod=0755 server/mediamtx/rover-snapshot-writer.sh /usr/local/bin/rover-snapshot-writer.sh
 

perf/nomad-benchmark.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""CPU benchmark of published NoMaD exploration, without ROS or rover connections.
+
+Run: python3 perf/nomad-benchmark.py --setup
+Subsequent runs: python3 perf/nomad-benchmark.py
+Quick run: python3 perf/nomad-benchmark.py --threads 4 --samples 1 --seconds 2 --iterations 2
+
+--setup installs CPU dependencies in a private cached venv and downloads pinned
+upstream source plus the official checkpoint. Nothing touches server packages.
+Synthetic images measure compute, NOT navigation quality. Excludes capture,
+video decode, network, physical control, training, and concurrent rover load.
+"""
+import argparse
+import hashlib
+import importlib.metadata
+import json
+import math
+import os
+from pathlib import Path
+import platform
+import resource
+import statistics
+import subprocess
+import sys
+import time
+import urllib.request
+import venv
+
+VINT = 'dca79815b704e5aa9c6bdc3082351f9e3b2848c2'
+DIFFUSION = '5ba07ac6661db573af695b419a7947ecb704690f'
+CHECKPOINT_ID = '1YJhkkMJAYOiKNyCaelbS_alpUpAJsOUb'
+SOURCES = {
+    ('robodhruv/visualnav-transformer', VINT): [
+        'train/config/nomad.yaml', 'train/vint_train/data/data_config.yaml',
+        'train/vint_train/models/nomad/nomad.py',
+        'train/vint_train/models/nomad/nomad_vint.py',
+        'train/vint_train/models/vint/self_attention.py',
+    ],
+    ('real-stanford/diffusion_policy', DIFFUSION): [
+        'diffusion_policy/model/diffusion/conditional_unet1d.py',
+        'diffusion_policy/model/diffusion/conv1d_components.py',
+        'diffusion_policy/model/diffusion/positional_embedding.py',
+    ],
+}
+
+
+def positive(value):
+    result = int(value)
+    if result < 1:
+        raise argparse.ArgumentTypeError('Must be positive')
+    return result
+
+
+def arguments():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('--setup', action='store_true')
+    p.add_argument('--cache', type=Path, default=Path.home() / '.cache/rover-nomad-benchmark')
+    p.add_argument('--threads', type=positive, nargs='+', default=[4, 8, 16])
+    p.add_argument('--samples', type=positive, nargs='+', default=[1, 8], help='Candidate trajectories per decision, not rover count')
+    p.add_argument('--seconds', type=positive, default=10)
+    p.add_argument('--iterations', type=positive, default=10)
+    p.add_argument('--worker', action='store_true', help=argparse.SUPPRESS)
+    p.add_argument('--result', type=Path, help=argparse.SUPPRESS)
+    return p.parse_args()
+
+
+def download(url, path):
+    if path.exists():
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + '.part')
+    print('Downloading', path.name, flush=True)
+    with urllib.request.urlopen(url, timeout=120) as response, temporary.open('wb') as output:
+        import shutil
+        shutil.copyfileobj(response, output)
+    temporary.replace(path)
+
+
+def prepare(args):
+    python = args.cache / 'venv/bin/python'
+    if not python.exists():
+        venv.EnvBuilder(with_pip=True).create(args.cache / 'venv')
+    subprocess.run([str(python), '-m', 'pip', 'install', 'torch', 'torchvision',
+                    '--index-url', 'https://download.pytorch.org/whl/cpu'], check=True)
+    subprocess.run([str(python), '-m', 'pip', 'install', 'numpy', 'diffusers',
+                    'efficientnet-pytorch', 'einops', 'PyYAML', 'gdown'], check=True)
+    for (repo, revision), paths in SOURCES.items():
+        for path in paths:
+            download(f'https://raw.githubusercontent.com/{repo}/{revision}/{path}', args.cache / 'source' / revision / path)
+    checkpoint = args.cache / 'nomad.pth'
+    if not checkpoint.exists():
+        temporary = checkpoint.with_suffix('.part')
+        subprocess.run([str(python), '-m', 'gdown', CHECKPOINT_ID, '-O', str(temporary)], check=True)
+        temporary.replace(checkpoint)
+
+
+def worker(args):
+    import numpy as np
+    import torch
+    import yaml
+    from diffusers.schedulers.scheduling_ddpm import DDPMScheduler
+    sys.path[:0] = [str(args.cache / 'source' / VINT / 'train'), str(args.cache / 'source' / DIFFUSION)]
+    from vint_train.models.nomad.nomad import NoMaD, DenseNetwork
+    from vint_train.models.nomad.nomad_vint import NoMaD_ViNT, replace_bn_with_gn
+    from diffusion_policy.model.diffusion.conditional_unet1d import ConditionalUnet1D
+
+    threads, samples = args.threads[0], args.samples[0]
+    torch.set_num_threads(threads)
+    torch.set_num_interop_threads(1)
+    torch.manual_seed(0)
+    c = yaml.safe_load((args.cache / 'source' / VINT / 'train/config/nomad.yaml').read_text())
+    vision = replace_bn_with_gn(NoMaD_ViNT(
+        context_size=c['context_size'], obs_encoding_size=c['encoding_size'],
+        mha_num_attention_heads=c['mha_num_attention_heads'],
+        mha_num_attention_layers=c['mha_num_attention_layers'], mha_ff_dim_factor=c['mha_ff_dim_factor']))
+    model = NoMaD(vision, ConditionalUnet1D(input_dim=2, global_cond_dim=c['encoding_size'],
+        down_dims=c['down_dims'], cond_predict_scale=c['cond_predict_scale']), DenseNetwork(c['encoding_size'])).eval()
+    # Strict loading prevents accidentally benchmarking a partly random model.
+    model.load_state_dict(torch.load(args.cache / 'nomad.pth', map_location='cpu', weights_only=True), strict=True)
+    scheduler = DDPMScheduler(num_train_timesteps=c['num_diffusion_iters'],
+        beta_schedule='squaredcos_cap_v2', clip_sample=True, prediction_type='epsilon')
+    width, height = c['image_size']
+    frames = torch.rand(c['context_size'] + 1, 3, height, width)
+    mean = torch.tensor([.485, .456, .406]).view(1, 3, 1, 1)
+    std = torch.tensor([.229, .224, .225]).view(1, 3, 1, 1)
+    mask = torch.ones(1, dtype=torch.long)
+    stats = yaml.safe_load((args.cache / 'source' / VINT / 'train/vint_train/data/data_config.yaml').read_text())['action_stats']
+    low, high = np.array(stats['min']), np.array(stats['max'])
+
+    def decision():
+        start = time.perf_counter()
+        obs = ((frames - mean) / std).reshape(1, -1, height, width)
+        goal = torch.randn(1, 3, height, width)
+        cond = model('vision_encoder', obs_img=obs, goal_img=goal, input_goal_mask=mask).repeat(samples, 1)
+        encoded = time.perf_counter()
+        action = torch.randn(samples, c['len_traj_pred'], 2)
+        scheduler.set_timesteps(c['num_diffusion_iters'])
+        for step in scheduler.timesteps:
+            noise = model('noise_pred_net', sample=action, timestep=step, global_cond=cond)
+            action = scheduler.step(noise, step, action).prev_sample
+        # Match upstream get_action: unnormalize deltas then accumulate waypoints.
+        waypoints = np.cumsum((action.numpy() + 1) / 2 * (high - low) + low, axis=1)
+        if not np.isfinite(waypoints).all():
+            raise RuntimeError('Non-finite model output')
+        end = time.perf_counter()
+        return (end - start) * 1000, (encoded - start) * 1000, (end - encoded) * 1000
+
+    rows = []
+    with torch.inference_mode():
+        for _ in range(3):
+            decision()
+        start, cpu = time.perf_counter(), time.process_time()
+        while len(rows) < args.iterations or time.perf_counter() - start < args.seconds:
+            rows.append(decision())
+        elapsed, cpu = time.perf_counter() - start, time.process_time() - cpu
+    total = sorted(r[0] for r in rows)
+    result = dict(threads=threads, candidateTrajectories=samples, iterations=len(rows),
+        p50Ms=statistics.median(total), p95Ms=total[math.ceil(.95 * len(total)) - 1], maxMs=max(total),
+        decisionsPerSecond=len(rows)/elapsed, cpuCores=cpu/elapsed,
+        peakMiB=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024,
+        visionMeanMs=statistics.mean(r[1] for r in rows), diffusionAndDecodeMeanMs=statistics.mean(r[2] for r in rows),
+        parameters=sum(p.numel() for p in model.parameters()), diffusionSteps=c['num_diffusion_iters'],
+        imageSize=c['image_size'], frames=c['context_size']+1,
+        versions={name: importlib.metadata.version(name) for name in ['torch','torchvision','diffusers','numpy','efficientnet-pytorch','einops']})
+    args.result.write_text(json.dumps(result))
+
+
+def main():
+    args = arguments()
+    args.cache = args.cache.expanduser().resolve()
+    if args.worker:
+        worker(args)
+        return
+    if args.setup:
+        prepare(args)
+    python = args.cache / 'venv/bin/python'
+    if not python.exists() or not (args.cache / 'nomad.pth').exists():
+        raise RuntimeError('First run: python3 perf/nomad-benchmark.py --setup')
+    print('CPU FP32 NoMaD exploration; synthetic images; no driving-quality measurement.', flush=True)
+    print('Candidates are trajectories for ONE rover. Includes vision + all 10 diffusion steps.', flush=True)
+    print('threads candidates  p50 ms  p95 ms  max ms decisions/s CPU cores peak MiB', flush=True)
+    results = []
+    import tempfile
+    with tempfile.TemporaryDirectory(prefix='nomad-bench-') as directory:
+        for threads in args.threads:
+            for samples in args.samples:
+                path = Path(directory) / 'result.json'
+                subprocess.run([str(python), str(Path(__file__).resolve()), '--worker', '--cache', str(args.cache),
+                    '--threads', str(threads), '--samples', str(samples), '--seconds', str(args.seconds),
+                    '--iterations', str(args.iterations), '--result', str(path)], check=True,
+                    timeout=max(600, args.seconds*20), env={**os.environ, 'PYTHONDONTWRITEBYTECODE':'1'})
+                r = json.loads(path.read_text()); results.append(r)
+                print(f"{threads:7} {samples:10} {r['p50Ms']:7.1f} {r['p95Ms']:7.1f} {r['maxMs']:7.1f} {r['decisionsPerSecond']:11.2f} {r['cpuCores']:9.2f} {r['peakMiB']:8.0f}", flush=True)
+    with (args.cache / 'nomad.pth').open('rb') as f:
+        digest = hashlib.file_digest(f, 'sha256').hexdigest()
+    report = dict(benchmark='nomad-cpu-v1', platform=platform.platform(), logicalCpus=os.cpu_count(),
+        loadAverage=os.getloadavg(), sourceRevisions=dict(vint=VINT,diffusion=DIFFUSION), checkpointSha256=digest,
+        exclusions=['capture','video decode','network','physical control','training','concurrent model workloads'], results=results)
+    print('\nPaste this report back:\n' + json.dumps(report, separators=(',', ':')))
+    print('At 10 decisions/s the total budget is 100ms; at 5/s it is 200ms. Leave room for the excluded pipeline.')
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except (Exception, KeyboardInterrupt) as error:
+        print(f'Benchmark failed: {error}', file=sys.stderr)
+        sys.exit(1)

server/index.js

@@ -68,6 +68,7 @@ require('./src/services/batteryManager');
 // subscriptions see fully decoded state without becoming an initialization
 // dependency of either control path.
 require('./src/services/fleetReportService');
+require('./src/services/roverLearningService');
 require('./src/services/replayEngineV2');
 // Replay delivery is a core service. It must subscribe before the optional
 // Discord feature so web requests always have a local delivery path.

server/src/configuration/configuration.test.js

@@ -184,6 +184,7 @@ test('service definitions generate public feature paths without a separate regis
     { key: 'discord', path: ['discord', 'enabled'] },
     { key: 'socials', path: ['socials', 'enabled'] },
     { key: 'fleetReports', path: ['fleetReports', 'enabled'] },
+    { key: 'roverLearning', path: ['roverLearning', 'enabled'] },
   ]);
 });
 

server/src/configuration/definition.js

@@ -22,6 +22,7 @@ const barcodeScanner = require('../services/barcodeScannerService/configuration'
 const commands = require('../services/operatorCommandService/configuration');
 const discord = require('../services/discordBotService/configuration');
 const fleetReports = require('../services/fleetReportService/configuration');
+const roverLearning = require('../services/roverLearningService/configuration');
 
 /*
   Object property order is preserved by JSON serialization and JSON Schema
@@ -55,6 +56,7 @@ const definitions = [
   sessionConfiguration.socials,
   sessionConfiguration.driverAd,
   fleetReports,
+  roverLearning,
 ];
 
 const defaultConfig = Object.fromEntries(

server/src/configuration/migrations.js

@@ -1,6 +1,6 @@
 // Configuration Database Migrations
 // Purpose: Applies ordered, transactional schema changes to the configuration and administration database.
-// Scope: Owns database structure only; configuration-document evolution belongs to the ordered definition and validation.
+// Scope: Upgrades database structure and stored configuration documents before current-schema validation.
 const migrations = [
   {
     version: 1,
@@ -72,6 +72,46 @@ const migrations = [
       });
     },
   },
+  {
+    version: 3,
+    run(db) {
+      // Migrate historical revisions too, so rolling back settings cannot
+      // restore the removed training-visit limit and break startup again.
+      db.prepare(`
+        UPDATE configuration_revisions
+        SET config_json = json_remove(config_json, '$.roverLearning.training.roundsPerSession')
+        WHERE json_type(config_json, '$.roverLearning.training.roundsPerSession') IS NOT NULL
+      `).run();
+    },
+  },
+  {
+    version: 4,
+    run(db) {
+      const rows = db.prepare('SELECT id, config_json FROM configuration_revisions').all();
+      const update = db.prepare('UPDATE configuration_revisions SET config_json = ? WHERE id = ?');
+      // Upgrade history as well as the active revision so a settings rollback
+      // cannot reintroduce fields rejected by the new Automations schema.
+      for (const row of rows) {
+        const config = JSON.parse(row.config_json);
+        let changed = false;
+        for (const item of config.homeAssistantActivities?.items || []) {
+          const hasAction = Object.hasOwn(item, 'idleAction');
+          const hasValue = Object.hasOwn(item, 'idleValue');
+          if (!hasAction && !hasValue) continue;
+          item.automations ??= {};
+          item.automations.idle ??= {};
+          // Preserve any explicitly saved new-format setting when both formats
+          // exist, including an empty text value used to clear an entity.
+          if (hasAction && !Object.hasOwn(item.automations.idle, 'action')) item.automations.idle.action = item.idleAction;
+          if (hasValue && !Object.hasOwn(item.automations.idle, 'value')) item.automations.idle.value = item.idleValue;
+          delete item.idleAction;
+          delete item.idleValue;
+          changed = true;
+        }
+        if (changed) update.run(JSON.stringify(config), row.id);
+      }
+    },
+  },
 ];
 
 function applySchemaMigrations(db) {

server/src/services/commandService/index.js

@@ -11,6 +11,7 @@ const logger = require('../../globals/logger').child('commandService');
 const { isHeadlightBlocked } = require('../../rewards/definitions/darkness');
 const homeAssistantService = require('../homeAssistantService');
 const overcurrentProtectionService = require('../overcurrentProtectionService');
+const { subscribeCommandRecording, observeCommand, recordCommandRequest } = require('./recording');
 
 // Command observations are intentionally separate from the global event bus.
 // Drive and motor commands can run at control-loop frequency, and publishing
@@ -96,6 +97,7 @@ function issueCommand(roverId, payload) {
   const normalizedPayload = normalizeOutboundCommandPayload(payload);
   const message = { ...normalizedPayload, id };
   record.ws.send(JSON.stringify(message));
+  observeCommand({ phase: 'dispatched', roverId, commandId: id, command: normalizedPayload });
   pendingCommands.set(id, { roverId, ts: Date.now(), type: normalizedPayload.type });
   logger.info('Issued command', roverId, normalizedPayload.type, id);
   return id;
@@ -130,6 +132,7 @@ function handleAck(msg) {
   const pending = pendingCommands.get(msg.id);
   if (!pending) return;
   pendingCommands.delete(msg.id);
+  observeCommand({ phase: 'acknowledged', roverId: pending.roverId, commandId: msg.id, acknowledgement: msg });
   logger.info('Command acknowledged', pending.roverId, pending.type, msg.status);
   io.emit('commandAck', {
     roverId: pending.roverId,
@@ -255,145 +258,149 @@ module.exports = {
   getRecentDriveActivity,
   setDriveCooldown,
   commandEvents,
+  subscribeCommandRecording,
+  submitCommand,
 };
 
-io.on('connection', (socket) => {
-  function handleCommand({ roverId, type, data } = {}, cb) {
-    const reply = typeof cb === 'function' ? cb : () => {};
-    try {
-      if (!roverId) {
-        throw new Error('roverId required');
+// Shared authorization/safety boundary for browser and service-owned control.
+function submitCommand(socket, { roverId, type, data } = {}, cb) {
+  const reply = typeof cb === 'function' ? cb : () => {};
+  try {
+    if (!roverId) {
+      throw new Error('roverId required');
+    }
+    if (!type) {
+      throw new Error('type required');
+    }
+    if (type === 'audioLevels') {
+      throw new Error('audioLevels command is service-managed');
+    }
+    let payload = data ? { ...data } : {};
+    if (type === 'headlight' && isHeadlightBlocked()) {
+      logger.info('Ignoring headlight command while darkness lock is active', { socketId: socket.id, roverId });
+      reply({ ignored: true, reason: 'darknessActive' });
+      return;
+    }
+    if (type === 'laser' && isLaserCommandBlockedByRoomLightLock(payload)) {
+      logger.info('Ignoring laser command while room lights are locked on', {
+        socketId: socket.id,
+        roverId,
+        action: getLaserAction(payload),
+      });
+      reply({ ignored: true, reason: 'roomLightsLockedOn' });
+      return;
+    }
+    const isRebootCommand = type === 'reboot';
+    const isUpdateCommand = type === 'update';
+    const isSongCommand = type === 'song' || (type === 'raw' && isSongRawPayload(payload));
+    const isAdminSocket = isAdmin(socket);
+    if (!isAdminSocket && isDeterred(socket)) {
+      throw new Error('Not authorized');
+    }
+    /*
+      Both structured song commands and raw Open Interface song payloads
+      reach this shared flag. Enforcing mute here covers the VIP MIDI beeper
+      and any future browser beeper without affecting unrelated driving.
+    */
+    if (!isAdminSocket && isSongCommand && isMuted(socket)) {
+      throw new Error('Muted');
+    }
+    // Rover updates run a privileged, root-owned helper on the Pi. Keep this
+    // in the same explicit admin-only branch as reboot instead of relying on
+    // drive ownership checks, because having a turn should not grant system
+    // maintenance privileges.
+    if ((isRebootCommand || isUpdateCommand) && !isAdminSocket) {
+      throw new Error('Not authorized');
+    }
+    if (!isSongCommand && !isRebootCommand && !isUpdateCommand && !roverManager.canDrive(roverId, socket)) {
+      throw new Error('Not your turn or no control');
+    }
+    const driveDirect = payload?.driveDirect;
+    if (type === 'drive' && driveDirect) {
+      if (!isAdminSocket) {
+        const safeDrive = roverManager.applyPrivateDriveSafety(roverId, socket, driveDirect);
+        if (safeDrive) {
+          payload.driveDirect = safeDrive;
+        }
       }
-      if (!type) {
-        throw new Error('type required');
-      }
-      if (type === 'audioLevels') {
-        throw new Error('audioLevels command is service-managed');
-      }
-      let payload = data ? { ...data } : {};
-      if (type === 'headlight' && isHeadlightBlocked()) {
-        logger.info('Ignoring headlight command while darkness lock is active', { socketId: socket.id, roverId });
-        reply({ ignored: true, reason: 'darknessActive' });
-        return;
+      const left = Number(payload?.driveDirect?.left);
+      const right = Number(payload?.driveDirect?.right);
+      const speed = Math.max(Math.abs(left), Math.abs(right));
+      if (!isAdminSocket) {
+        const blockedUntil = driveCooldowns.get(roverId);
+        if (blockedUntil && Date.now() < blockedUntil && speed > 0) {
+          const reason = isLockdownAdmin(socket)
+            ? 'Drive blocked: cooldown'
+            : 'Drive blocked: safety cooldown';
+          throw new Error(reason);
+        }
       }
-      if (type === 'laser' && isLaserCommandBlockedByRoomLightLock(payload)) {
-        logger.info('Ignoring laser command while room lights are locked on', {
+      if (speed > 0) {
+        let direction = 'turn';
+        if (left > 0 && right > 0) direction = 'forward';
+        if (left < 0 && right < 0) direction = 'backward';
+        lastDriveActivity.set(roverId, {
+          ts: Date.now(),
           socketId: socket.id,
-          roverId,
-          action: getLaserAction(payload),
-        });
-        reply({ ignored: true, reason: 'roomLightsLockedOn' });
-        return;
-      }
-      const isRebootCommand = type === 'reboot';
-      const isUpdateCommand = type === 'update';
-      const isSongCommand = type === 'song' || (type === 'raw' && isSongRawPayload(payload));
-      const isAdminSocket = isAdmin(socket);
-      if (!isAdminSocket && isDeterred(socket)) {
-        throw new Error('Not authorized');
-      }
-      /*
-        Both structured song commands and raw Open Interface song payloads
-        reach this shared flag. Enforcing mute here covers the VIP MIDI beeper
-        and any future browser beeper without affecting unrelated driving.
-      */
-      if (!isAdminSocket && isSongCommand && isMuted(socket)) {
-        throw new Error('Muted');
-      }
-      // Rover updates run a privileged, root-owned helper on the Pi. Keep this
-      // in the same explicit admin-only branch as reboot instead of relying on
-      // drive ownership checks, because having a turn should not grant system
-      // maintenance privileges.
-      if ((isRebootCommand || isUpdateCommand) && !isAdminSocket) {
-        throw new Error('Not authorized');
-      }
-      if (!isSongCommand && !isRebootCommand && !isUpdateCommand && !roverManager.canDrive(roverId, socket)) {
-        throw new Error('Not your turn or no control');
-      }
-      const driveDirect = payload?.driveDirect;
-      if (type === 'drive' && driveDirect) {
-        if (!isAdminSocket) {
-          const safeDrive = roverManager.applyPrivateDriveSafety(roverId, socket, driveDirect);
-          if (safeDrive) {
-            payload.driveDirect = safeDrive;
-          }
-        }
-        const left = Number(payload?.driveDirect?.left);
-        const right = Number(payload?.driveDirect?.right);
-        const speed = Math.max(Math.abs(left), Math.abs(right));
-        if (!isAdminSocket) {
-          const blockedUntil = driveCooldowns.get(roverId);
-          if (blockedUntil && Date.now() < blockedUntil && speed > 0) {
-            const reason = isLockdownAdmin(socket)
-              ? 'Drive blocked: cooldown'
-              : 'Drive blocked: safety cooldown';
-            throw new Error(reason);
-          }
-        }
-        if (speed > 0) {
-          let direction = 'turn';
-          if (left > 0 && right > 0) direction = 'forward';
-          if (left < 0 && right < 0) direction = 'backward';
-          lastDriveActivity.set(roverId, {
-            ts: Date.now(),
-            socketId: socket.id,
-            direction,
-            speed,
-            isAdmin: isAdminSocket,
-          });
-        }
-      }
-
-      if (type === 'drive' || type === 'motors') {
-        /*
-          Role is supplied at the command boundary because telemetry does not
-          identify the operator who produced the active motor intent. Admin and
-          lockdown commands therefore enter the service explicitly bypassed;
-          they are recorded for status visibility but are never scaled, blocked,
-          or countermanded by a later sensor frame.
-        */
-        payload = overcurrentProtectionService.protectCommand(roverId, type, payload, {
-          bypassed: isAdminSocket,
+          direction,
+          speed,
+          isAdmin: isAdminSocket,
         });
       }
-      const id = issueCommand(roverId, { type, ...payload });
-      commandEvents.emit('observation', {
-        ts: Date.now(),
-        roverId: String(roverId),
-        type,
-        commandId: id,
-        outcome: 'issued',
-        socketId: socket.id,
-        // Payloads are omitted deliberately: raw OI, TTS, and maintenance
-        // commands can carry arbitrary content. Their structured type/outcome
-        // supplies analytics without accidentally persisting secret material.
-      });
-      logger.info('Queued command', socket.id, roverId, type);
-      if (shouldRecordTurnActivity(type, payload)) {
-        try {
-          const { recordActivity } = require('../turnService');
-          recordActivity(roverId, socket.id);
-        } catch (err) {
-          // Activity recording is best-effort because command delivery should not fail if turn bookkeeping has a transient issue.
-        }
-      }
-      reply({ id });
-    } catch (err) {
-      logger.warn('Command rejected', socket.id, err.message);
-      commandEvents.emit('observation', {
-        ts: Date.now(),
-        roverId: roverId ? String(roverId) : null,
-        type: type || 'unknown',
-        outcome: 'rejected',
-        socketId: socket.id,
-        error: err.message,
-      });
-      reply({ error: err.message });
     }
-  }
 
-  socket.on('command', handleCommand);
-  socket.on('command:issue', handleCommand);
+    if (type === 'drive' || type === 'motors') {
+      /*
+        Role is supplied at the command boundary because telemetry does not
+        identify the operator who produced the active motor intent. Admin and
+        lockdown commands therefore enter the service explicitly bypassed;
+        they are recorded for status visibility but are never scaled, blocked,
+        or countermanded by a later sensor frame.
+      */
+      payload = overcurrentProtectionService.protectCommand(roverId, type, payload, {
+        bypassed: isAdminSocket,
+      });
+    }
+    const id = issueCommand(roverId, { type, ...payload });
+    commandEvents.emit('observation', {
+      ts: Date.now(),
+      roverId: String(roverId),
+      type,
+      commandId: id,
+      outcome: 'issued',
+      socketId: socket.id,
+      // Payloads are omitted deliberately: raw OI, TTS, and maintenance
+      // commands can carry arbitrary content. Their structured type/outcome
+      // supplies analytics without accidentally persisting secret material.
+    });
+    logger.info('Queued command', socket.id, roverId, type);
+    if (shouldRecordTurnActivity(type, payload)) {
+      try {
+        const { recordActivity } = require('../turnService');
+        recordActivity(roverId, socket.id);
+      } catch (err) {
+        // Activity recording is best-effort because command delivery should not fail if turn bookkeeping has a transient issue.
+      }
+    }
+    reply({ id });
+  } catch (err) {
+    logger.warn('Command rejected', socket.id, err.message);
+    commandEvents.emit('observation', {
+      ts: Date.now(),
+      roverId: roverId ? String(roverId) : null,
+      type: type || 'unknown',
+      outcome: 'rejected',
+      socketId: socket.id,
+      error: err.message,
+    });
+    reply({ error: err.message });
+  }
+}
+
+io.on('connection', (socket) => {
+  const recordedCommand = (request, reply) => recordCommandRequest(socket, request, reply, (message, cb) => submitCommand(socket, message, cb));
+  socket.on('command', recordedCommand);
+  socket.on('command:issue', recordedCommand);
 
   socket.on('command:updateAllRovers', (_payload = {}, cb) => {
     const reply = typeof cb === 'function' ? cb : () => {};

server/src/services/commandService/recording.js

@@ -0,0 +1,55 @@
+// Optional detailed command observation, separate from payload-free analytics.
+// Subscribers must only enqueue work; recorder failures cannot fail a command.
+const { AsyncLocalStorage } = require('node:async_hooks');
+const { randomUUID } = require('node:crypto');
+
+const observers = new Set();
+const requests = new AsyncLocalStorage();
+
+function subscribeCommandRecording(observer) {
+  observers.add(observer);
+  return () => observers.delete(observer);
+}
+
+function observeCommand(event) {
+  if (!observers.size) return;
+  const scope = requests.getStore();
+  const context = scope?.active ? scope.context : null;
+  for (const observer of observers) {
+    try {
+      // Snapshot before safety code or a caller mutates a nested payload. Each
+      // subscriber receives its own copy and cannot change command delivery.
+      observer(structuredClone({
+        ts: Date.now(),
+        monotonicNs: process.hrtime.bigint().toString(),
+        source: context ? 'client' : 'server',
+        ...context,
+        ...event,
+      }));
+    } catch {
+      // Recording is deliberately best-effort at this boundary. The owning
+      // recorder reports its storage health independently of driver replies.
+    }
+  }
+}
+
+function recordCommandRequest(socket, request, reply, execute, source = 'client') {
+  if (!observers.size) return execute(request, reply);
+  const context = { requestId: randomUUID(), socketId: socket.id, userId: socket.data?.userId || null, source };
+  const scope = { active: true, context };
+  return requests.run(scope, () => {
+    try {
+      observeCommand({ phase: 'requested', roverId: request?.roverId, command: request });
+      return execute(request, (result) => {
+        observeCommand({ phase: 'result', roverId: request?.roverId, result });
+        if (typeof reply === 'function') reply(result);
+      });
+    } finally {
+      // The command handler is synchronous. A safety timer created inside it
+      // inherits async context but its later commands are server actions.
+      scope.active = false;
+    }
+  });
+}
+
+module.exports = { subscribeCommandRecording, observeCommand, recordCommandRequest };

server/src/services/homeAssistantActivitiesService/actions.js

@@ -1,5 +1,4 @@
-// User and idle commands share one dispatch contract, while only the trusted
-// idle entry point can bypass per-item user locks.
+// User and automated commands share validation; only trusted automation bypasses user permissions.
 const { buildEntity, buildCommand } = require('./entityHelpers');
 
 function assertAccess(actor = {}) {
@@ -9,13 +8,14 @@ function assertAccess(actor = {}) {
 }
 
 function createActions({ getConfig, ha, locks }) {
-  async function execute(id, value, actor, idle = false) {
-    if (!idle) assertAccess(actor);
+  async function execute(id, value, actor, automated = false) {
+    if (!automated) assertAccess(actor);
     const config = getConfig();
     if (!config.enabled) throw new Error('Activity Controls are disabled');
     const item = config.items.find((entry) => entry.id === id);
     if (!item) throw new Error('Unknown activity item');
-    if (!idle && locks.isLocked(id) && !['admin', 'lockdown'].includes(actor.role)) throw new Error('This item is locked');
+    if (!automated && locks.isLocked(id) && !['admin', 'lockdown'].includes(actor.role)) throw new Error('This item is locked');
+    if (!automated && item.readOnly && !['admin', 'lockdown'].includes(actor.role)) throw new Error('This item is read-only for users');
     if (!ha.enabled || !ha.isConnected()) throw new Error('Home Assistant is offline');
     const entity = buildEntity(item, ha.getRawEntitySnapshot(id));
     if (!entity.available) throw new Error('This item is unavailable');
@@ -25,30 +25,34 @@ function createActions({ getConfig, ha, locks }) {
     await ha.callHomeAssistantService(entity.domain, command.service, command.data);
   }
 
-  async function runIdleActions() {
+  async function runAutomation(trigger, shouldRun = () => true) {
     const config = getConfig();
-    if (!config.enabled) return { action: 'homeAssistantActivitiesIdle', skipped: true };
+    if (!config.enabled) return { action: trigger, skipped: true };
     const results = [];
     // Catch per item so one offline integration cannot prevent the remaining
-    // configured cleanup actions from running during this idle window.
+    // configured actions from running for this trigger.
     for (const item of config.items) {
-      if (!item.idleAction || item.idleAction === 'unchanged' || item.readOnly) continue;
+      // Stop remaining writes if presence or mode changed while HA was responding.
+      if (!shouldRun()) break;
+      const setting = item.automations?.[trigger];
+      if (!setting?.action || setting.action === 'unchanged') continue;
       try {
         const entity = buildEntity(item, ha.getRawEntitySnapshot(item.id));
         if (entity.type === 'readOnly') continue;
-        if ((item.idleAction === 'press') !== (entity.type === 'button')) throw new Error('Idle action does not match the control type');
+        if ((setting.action === 'press') !== (entity.type === 'button')) throw new Error('Automation action does not match the control type');
         // The admin form omits an empty optional string. Treat that as empty
         // text so clearing a message works; other types still reject it through
         // their normal value validation rather than silently receiving zero.
-        await execute(item.id, item.idleAction === 'press' ? 'press' : (item.idleValue ?? ''), null, true);
+        await execute(item.id, setting.action === 'press' ? 'press' : (setting.value ?? ''), null, true);
         results.push({ id: item.id, ok: true });
       } catch (error) {
         results.push({ id: item.id, ok: false, error: error.message });
       }
     }
-    return { action: 'homeAssistantActivitiesIdle', results };
+    return { action: trigger, results };
   }
-  return { act: (id, value, actor) => execute(id, value, actor), runIdleActions };
+  return { act: (id, value, actor) => execute(id, value, actor), runAutomation,
+    runIdleActions: () => runAutomation('idle') };
 }
 
 module.exports = { createActions, assertAccess };

server/src/services/homeAssistantActivitiesService/activities.test.js

@@ -29,7 +29,7 @@ test('sensor states, live constraints, and unknown button states are preserved',
   assert.equal(buildEntity({ id: 'button.bell' }, raw('unknown')).available, true);
   assert.equal(buildEntity({ id: 'button.bell' }, raw('unavailable')).available, false);
   assert.equal(buildEntity({ id: 'button.bell' }, null).available, false);
-  assert.equal(buildEntity({ id: 'switch.fan', readOnly: true }, raw('on')).type, 'readOnly');
+  assert.equal(buildEntity({ id: 'switch.fan', readOnly: true }, raw('on')).type, 'toggle');
 });
 
 test('domain dispatch handles native entities and helpers without room-light payloads', async () => {
@@ -87,22 +87,22 @@ test('permissions, locks, availability and allowlist are authoritative', async (
 test('idle ignores user locks, preserves no-op items, and isolates failures', async () => {
   const h = harness([
     { id: 'switch.keep' },
-    { id: 'number.bad', idleAction: 'set', idleValue: '200' },
-    { id: 'switch.fan', idleAction: 'set', idleValue: 'off' },
+    { id: 'number.bad', automations: { idle: { action: 'set', value: '200' } } },
+    { id: 'switch.fan', automations: { idle: { action: 'set', value: 'off' } } },
     // An empty optional text box is omitted by the schema-driven admin form.
-    { id: 'input_text.message', idleAction: 'set' },
-    { id: 'input_button.stop', idleAction: 'press' },
-    { id: 'sensor.humidity', idleAction: 'set', idleValue: '0' },
-    { id: 'switch.readonly', readOnly: true, idleAction: 'set', idleValue: 'off' },
+    { id: 'input_text.message', automations: { idle: { action: 'set' } } },
+    { id: 'input_button.stop', automations: { idle: { action: 'press' } } },
+    { id: 'sensor.humidity', automations: { idle: { action: 'set', value: '0' } } },
+    { id: 'switch.readonly', readOnly: true, automations: { idle: { action: 'set', value: 'off' } } },
   ], {
-    'number.bad': raw('0', { min: 0, max: 10 }), 'switch.fan': raw('on'),
+    'switch.readonly': raw('on'), 'number.bad': raw('0', { min: 0, max: 10 }), 'switch.fan': raw('on'),
     'input_text.message': raw('Welcome', { min: 0, max: 30 }), 'input_button.stop': raw('unknown'),
   });
   h.locked.add('switch.fan');
   const result = await h.actions.runIdleActions();
   assert.equal(result.results[0].ok, false);
-  assert.equal(result.results.filter((entry) => entry.ok).length, 3);
-  assert.deepEqual(h.calls.map((call) => call[2]), [{ entity_id: 'switch.fan' }, { entity_id: 'input_text.message', value: '' }, { entity_id: 'input_button.stop' }]);
+  assert.equal(result.results.filter((entry) => entry.ok).length, 4);
+  assert.deepEqual(h.calls.map((call) => call[2]), [{ entity_id: 'switch.fan' }, { entity_id: 'input_text.message', value: '' }, { entity_id: 'input_button.stop' }, { entity_id: 'switch.readonly' }]);
 });
 
 test('an outstanding service response does not block later commands', async () => {

server/src/services/homeAssistantActivitiesService/configuration.js

@@ -2,6 +2,14 @@
 // never acquire unrelated devices simply because they share a connection.
 const { strictObject, string, boolean } = require('../../configuration/schemaHelpers');
 
+// All triggers use the same inputs so the admin form and execution contract agree.
+function automation(title, description) {
+  return strictObject({
+    action: string({ title: 'Action', enum: ['unchanged', 'set', 'press'], default: 'unchanged' }),
+    value: string({ title: 'Value', description: 'For set: on/off, a number, exact selection, or text. Empty text clears it. Ignored for press.', maxLength: 255 }),
+  }, { title, description });
+}
+
 module.exports = {
   key: 'homeAssistantActivities',
   feature: true,
@@ -18,9 +26,12 @@ module.exports = {
         icon: string({ description: 'Font Awesome name, as used by social links. Blank or invalid names use a fallback.', maxLength: 80, examples: ['FaFan'] }),
         // Match social-link colors so admins can customize tiles through the existing config form.
         color: string({ description: 'Optional six-digit hexadecimal tile color, like social links.', examples: ['#3B82F6'], pattern: '^#[0-9a-fA-F]{6}$' }),
-        readOnly: boolean({ title: 'Read only', default: false, description: 'Display the value without allowing user or idle commands.' }),
-        idleAction: string({ title: 'When idle', enum: ['unchanged', 'set', 'press'], default: 'unchanged', description: 'Leave unchanged, set the idle value, or press a button once. Runs independently of user locks.' }),
-        idleValue: string({ title: 'Idle value', description: 'For set: on/off, a number, exact selection, or text (empty text clears it). Checked against live entity limits when idle runs.', examples: ['0'], maxLength: 255 }),
+        readOnly: boolean({ title: 'Read only for users', default: false, description: 'Ordinary users can only view this item. Admins and automations can still change supported controls.' }),
+        automations: strictObject({
+          idle: automation('When idle', 'Runs after the idle timeout with no users or admins online.'),
+          locked: automation('When server locked', 'Runs on entering admin or lockdown mode.'),
+          online: automation('When users online', 'Runs when a user or admin comes online, or the server reopens with one online. Suppressed in admin and lockdown modes. Spectators do not count.'),
+        }, { title: 'Automations', description: 'One-time actions that bypass item locks and read-only permissions.' }),
       }, { description: 'One independently controlled or read-only activity control item.', required: ['id'] }),
     },
   }, { title: 'Activity Controls', description: 'Generic activity controls and idle behavior, separate from room lighting.', required: ['enabled', 'items'] }),

server/src/services/homeAssistantActivitiesService/entityHelpers.js

@@ -9,7 +9,8 @@ const TYPES = {
 function buildEntity(item, raw, locked = false) {
   const attributes = raw?.attributes || {};
   const domain = item.id.split('.')[0];
-  const type = item.readOnly ? 'readOnly' : TYPES[domain] || 'readOnly';
+  // Read-only is a user permission, not an entity capability: admins and automation still need its write type.
+  const type = TYPES[domain] || 'readOnly';
   // A never-pressed button legitimately reports unknown. Its state is a last
   // press timestamp, not availability or a boolean toggle state.
   const available = Boolean(raw && raw.state !== 'unavailable' && (raw.state !== 'unknown' || type === 'button'));
@@ -17,7 +18,7 @@ function buildEntity(item, raw, locked = false) {
   return {
     id: item.id, name: item.name?.trim() || attributes.friendly_name || item.id,
     // Include presentation settings in session state so all clients share the configured color.
-    icon: item.icon || '', color: item.color || '', domain, type, locked, available,
+    icon: item.icon || '', color: item.color || '', domain, type, readOnly: Boolean(item.readOnly), locked, available,
     state: raw?.state ?? 'unknown', unit: attributes.unit_of_measurement || '',
     min: numeric(attributes.min), max: numeric(attributes.max), step: numeric(attributes.step),
     options: Array.isArray(attributes.options) ? attributes.options.filter((option) => typeof option === 'string') : [],

server/src/services/homeAssistantActivitiesService/index.js

@@ -8,7 +8,7 @@ const { buildEntity } = require('./entityHelpers');
 const { createLocks, resolveItem } = require('./locks');
 const { createActions } = require('./actions');
 const { getRole } = require('../roleService');
-const { getMode } = require('../modeManager');
+const { getMode, modeEvents } = require('../modeManager');
 const logger = require('../../globals/logger').child('homeAssistantActivitiesService');
 
 const events = new EventEmitter();
@@ -24,6 +24,35 @@ const actions = createActions({ getConfig: () => config, ha: {
   callHomeAssistantService: ha.callHomeAssistantService,
 }, locks });
 
+let operatorsOnline = false;
+let lastAutomationState = null;
+let automationRevision = 0;
+
+function refreshAutomation() {
+  const mode = getMode();
+  const locked = mode === 'admin' || mode === 'lockdown';
+  // Remember transitions, not entity values: HA updates and extra connections
+  // must not repeatedly press buttons or undo a manual admin adjustment.
+  const state = config.enabled && snapshotReady && ha.isConnected()
+    ? (locked ? mode : operatorsOnline ? 'online' : null) : null;
+  if (state === lastAutomationState) return;
+  lastAutomationState = state;
+  const revision = ++automationRevision;
+  if (!state) return;
+  actions.runAutomation(locked ? 'locked' : 'online', () => revision === automationRevision)
+    .then((result) => {
+      const failures = result.results?.filter((item) => !item.ok) || [];
+      if (failures.length) logger.warn('Activity automation failed', { trigger: result.action, failures });
+    }).catch((error) => logger.warn('Activity automation failed', { error: error.message }));
+}
+
+function setOperatorsOnline(online) {
+  operatorsOnline = online;
+  refreshAutomation();
+}
+
+modeEvents.on('change', refreshAutomation);
+
 function getState() {
   return {
     enabled: config.enabled,
@@ -53,14 +82,17 @@ function setLocked(query, locked) {
 ha.homeAssistantEvents.on('snapshot', () => {
   snapshotReady = true;
   emitUpdate();
+  refreshAutomation();
 });
 ha.homeAssistantEvents.on('status', () => {
   snapshotReady = false;
   emitUpdate();
+  refreshAutomation();
 });
 registerConfigurationHandler('homeAssistantActivities', (next) => {
   config = next;
   emitUpdate();
+  refreshAutomation();
 });
 // Resolve access on the server for every command; clients only supply item/value.
 io.on('connection', (socket) => {
@@ -76,4 +108,7 @@ io.on('connection', (socket) => {
   });
 });
 
-module.exports = { ...actions, getState, setLocked, activityEvents: events };
+module.exports = { ...actions, getState, setLocked, setOperatorsOnline, activityEvents: events,
+  // A returning operator ends idle cleanup even if earlier HA calls are still settling.
+  runIdleActions: () => actions.runAutomation('idle', () => !operatorsOnline),
+};

server/src/services/idleService/index.js

@@ -7,6 +7,7 @@ const { getRole, roleEvents } = require('../roleService');
 const { IDLE_TIMEOUT_MS } = require('./constants');
 const { runtime } = require('./state');
 const { runIdleActions } = require('./actions');
+const activityControls = require('../homeAssistantActivitiesService');
 
 function getActivitySnapshot() {
   let onlineUsers = 0;
@@ -113,6 +114,8 @@ function scheduleIdleTimer() {
 function refreshIdleState() {
   const activity = getActivitySnapshot();
   logger.info('Idle state refresh', activity);
+  // Share the exact idle presence policy; Activity Controls must not count spectators or invent a second role filter.
+  activityControls.setOperatorsOnline(activity.totalActive > 0);
   if (activity.totalActive > 0) {
     clearIdleTimer();
     if (runtime.idleActionsCompleted) {

server/src/services/roomCameraService/configuration.js

@@ -1,7 +1,7 @@
 // Room-Camera Configuration
 // Purpose: Defines the optional named snapshot and stream camera catalog.
 // Scope: Contains configuration metadata only and never contacts a camera.
-const { strictObject, string, boolean } = require('../../configuration/schemaHelpers');
+const { strictObject, string, boolean, integer } = require('../../configuration/schemaHelpers');
 
 module.exports = {
   key: 'roomCameras',
@@ -10,6 +10,7 @@ module.exports = {
   // initial configuration while the feature switch prevents network requests.
   defaultValue: {
     enabled: false,
+    streamIntervalMs: 2000,
     cameras: [
       {
         id: 'lobby',
@@ -29,6 +30,7 @@ module.exports = {
   },
   schema: strictObject({
     enabled: boolean({ description: 'Immediately publishes the configured room-camera catalog and enables camera snapshots and streams.' }),
+    streamIntervalMs: integer({ title: 'Stream interval (ms)', description: 'Interval for sending room camera images to clients. Lowering this increases bandwidth usage FAST because it sends a full image each time.', minimum: 1 }),
     cameras: {
       type: 'array',
       description: 'Room cameras available to the web UI and replay system.',

server/src/services/roomCameraService/index.js

@@ -8,10 +8,12 @@ const replay = require('../replayEngineV2/roomCameraReplayBuilder');
 const { loadConfig, registerConfigurationHandler } = require('../../configuration');
 
 let enabled = false;
+let streamIntervalMs = 2000;
 
 const snapshotEngine = createSnapshotEngine({ getRoomCameras, roomCameraEvents });
 function applyRoomCameraConfig(roomCameraConfig = {}) {
   enabled = Boolean(roomCameraConfig.enabled);
+  streamIntervalMs = roomCameraConfig.streamIntervalMs ?? 2000;
   // Loading an empty catalog on disable causes the snapshot engine's existing
   // update listener to close every stream and timer without unregistering the
   // stable browser gateway.
@@ -19,6 +21,7 @@ function applyRoomCameraConfig(roomCameraConfig = {}) {
 }
 
 registerRoomCameraSocketGateway({
+  getStreamIntervalMs: () => streamIntervalMs,
   getRoomCamera,
   getRoomCameras,
   getRoomCameraState: snapshotEngine.getRoomCameraState,

server/src/services/roomCameraService/socketGateway.js

@@ -8,7 +8,6 @@ const { isAdmin, isLockdownAdmin, getRole } = require('../roleService');
 
 const SUBSCRIBE_LIMIT = 50;
 const SUBSCRIBE_WINDOW_MS = 10000;
-const STREAM_INTERVAL_MS = 2000;
 
 function passesMode(socket) {
   const mode = getMode();
@@ -20,7 +19,7 @@ function passesMode(socket) {
   return true;
 }
 
-function registerRoomCameraSocketGateway({ getRoomCamera, getRoomCameras, getRoomCameraState, roomCameraStreamEvents }) {
+function registerRoomCameraSocketGateway({ getRoomCamera, getRoomCameras, getRoomCameraState, roomCameraStreamEvents, getStreamIntervalMs }) {
   const cameraSubscribers = new Map();
   const socketSubscriptions = new Map();
   const subscribeBuckets = new Map();
@@ -82,7 +81,7 @@ function registerRoomCameraSocketGateway({ getRoomCamera, getRoomCameras, getRoo
       }
       const lastSent = lastMap.get(id) || 0;
       const now = ts || Date.now();
-      if (now - lastSent < STREAM_INTERVAL_MS) return;
+      if (now - lastSent < getStreamIntervalMs()) return;
       lastMap.set(id, now);
       sendFrame(socket, id, { ts }, buffer);
     });

server/src/services/roverLearningService/capabilities.js

@@ -0,0 +1,18 @@
+// Match actuator configuration, not a rover's display name or changing telemetry.
+const { isDeepStrictEqual } = require('node:util');
+const POLICY_VERSION = 5;
+
+function controlProfile(meta = {}) {
+  return Object.fromEntries(['cameraServo', 'headlight', 'roomba', 'platform']
+    .map((key) => [key, meta?.[key] ?? null]));
+}
+
+function compatible(model, record) {
+  if (model?.specification?.version !== POLICY_VERSION || !record) return false;
+  const profile = controlProfile(record.meta);
+  // Retained recordings may include removed actuator fields. Compare only
+  // controls this policy actually uses, on both sides of the boundary.
+  return Array.isArray(model.controlProfiles) && model.controlProfiles.some((saved) => isDeepStrictEqual(controlProfile(saved), profile));
+}
+
+module.exports = { POLICY_VERSION, controlProfile, compatible };

server/src/services/roverLearningService/configuration.js

@@ -0,0 +1,47 @@
+// Metadata only: loading the configuration must never start the learning runtime.
+const { strictObject, boolean, integer, number, string } = require('../../configuration/schemaHelpers');
+
+module.exports = {
+  key: 'roverLearning',
+  feature: true,
+  defaultValue: {
+    enabled: false,
+    recording: { maxGiB: 20, minimumFreeGiB: 2, sessionSeconds: 300 },
+    driving: { threads: 4, actionThreshold: 0.7, staleMs: 2000 },
+    training: {
+      enabled: true, python: '/opt/rover-learning/bin/python', threads: 8,
+      batchSize: 16, maxSamples: 1024, minimumSamples: 128, passesPerJob: 2,
+      maxStepsPerJob: 200, checkpointEverySteps: 1000,
+      minimumDrivingMinutes: 30, newDrivingMinutesPerModel: 10,
+      intervalSeconds: 60, maxJobSeconds: 600,
+    },
+  },
+  schema: strictObject({
+    enabled: boolean({ description: 'Enable human-session recording, background learning, and verified-user model control. Disabled starts no workers or storage.' }),
+    driving: strictObject({
+      threads: integer({ minimum: 1, maximum: 16, description: 'CPU threads per active model controller.' }),
+      actionThreshold: number({ minimum: 0.5, maximum: 0.99, description: 'Minimum model score for issuing a command, including drive and stop commands.' }),
+      staleMs: integer({ minimum: 1000, maximum: 5000, description: 'Stop when video, sensors, or model predictions are older than this many milliseconds.' }),
+    }, { description: 'Live model execution. Manual inputs do not stop or pause the model.', required: ['threads', 'actionThreshold', 'staleMs'] }),
+    recording: strictObject({
+      maxGiB: number({ minimum: 0.1, maximum: 100000, description: 'Rolling recording budget in GiB. Checked every 15 seconds; active files can temporarily exceed it. Published models are never pruned.' }),
+      minimumFreeGiB: number({ minimum: 0.1, maximum: 100000, description: 'Pause recording below this amount of available filesystem space, in GiB.' }),
+      sessionSeconds: integer({ minimum: 30, maximum: 3600, description: 'Rotate video and telemetry sessions after this many seconds so completed sessions can be pruned.' }),
+    }, { description: 'Bounded recording storage. Limits never delete published models.', required: ['maxGiB', 'minimumFreeGiB', 'sessionSeconds'] }),
+    training: strictObject({
+      enabled: boolean({ description: 'Train periodically from completed usable human recordings while the service is enabled.' }),
+      python: string({ minLength: 1, description: 'Python executable with the service requirements installed. The container supplies /opt/rover-learning/bin/python.' }),
+      threads: integer({ minimum: 1, maximum: 32, description: 'CPU threads for the lower-priority training process.' }),
+      batchSize: integer({ minimum: 1, maximum: 64, description: 'Training windows per optimizer step.' }),
+      maxSamples: integer({ minimum: 128, maximum: 2048, description: 'Maximum decoded windows retained in RAM per job. Each contains four 160x120 RGB frames.' }),
+      minimumSamples: integer({ minimum: 1, maximum: 128, description: 'Usable sampled windows required before training.' }),
+      passesPerJob: integer({ minimum: 1, maximum: 10, description: 'Shuffled passes over the sampled windows per job.' }),
+      minimumDrivingMinutes: number({ minimum: 1, maximum: 100000, description: 'Minimum distinct usable training footage in minutes before promoting Latest; repeated passes do not count.' }),
+      newDrivingMinutesPerModel: number({ minimum: 1, maximum: 100000, description: 'Additional distinct usable footage required between Latest promotions.' }),
+      maxStepsPerJob: integer({ minimum: 1, maximum: 10000, description: 'Optimizer step cap per finite training job.' }),
+      checkpointEverySteps: integer({ minimum: 1, maximum: 100000, description: 'Accumulated training steps between gated Latest promotions. Permanent named snapshots are created by administrators.' }),
+      intervalSeconds: integer({ minimum: 30, maximum: 86400, description: 'Minimum seconds between completed training jobs.' }),
+      maxJobSeconds: integer({ minimum: 30, maximum: 3600, description: 'Wall-clock budget including video decoding; slow/stuck workers are stopped.' }),
+    }, { description: 'Ongoing CPU learning, sampling, and gated Latest checkpoint updates.', required: ['enabled', 'python', 'threads', 'batchSize', 'maxSamples', 'minimumSamples', 'passesPerJob', 'minimumDrivingMinutes', 'newDrivingMinutesPerModel', 'maxStepsPerJob', 'checkpointEverySteps', 'intervalSeconds', 'maxJobSeconds'] }),
+  }, { description: 'Optional rover recording and learning service.', required: ['enabled', 'recording', 'training', 'driving'] }),
+};

server/src/services/roverLearningService/diagnostics.js

@@ -0,0 +1,64 @@
+// Export an explicit allowlist, never the raw socket/session/configuration object.
+const { redact } = require('./errors');
+const { POLICY_VERSION } = require('./capabilities');
+const os = require('node:os');
+const fs = require('node:fs');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const applicationVersion = require('../../../package.json').version;
+const hash = createHash('sha256');
+for (const file of ['workers/policy.py', 'workers/dataset.py', 'workers/train.py', 'workers/infer.py', 'workers/observations.py', 'workers/learning.py',
+  'capabilities.js', 'errors.js', 'storage.js', 'runtime.js', 'models.js', 'driving.js', 'recording.js', 'training.js', 'socketGateway.js', 'diagnostics.js']) {
+  hash.update(fs.readFileSync(path.join(__dirname, file)));
+}
+const implementationSha256 = hash.digest('hex');
+
+function errorCategory(value) {
+  if (!value) return null;
+  const text = String(value).toLowerCase();
+  for (const [pattern, category] of [
+    [/timestamp|pts|clock|aligned/, 'time_alignment'], [/sensor/, 'sensors'],
+    [/video|decode|ffmpeg|rtsp/, 'video'], [/permission|turn|control permission/, 'permission'],
+    [/schema|shape|version/, 'model_schema'], [/space|disk|enospc/, 'storage'],
+    [/sample|window|command/, 'dataset_or_command'], [/worker|exited|process/, 'worker'],
+  ]) if (pattern.test(text)) return category;
+  return 'unclassified';
+}
+
+function report(state, controller) {
+  const training = state.training || {};
+  const metrics = training.diagnostics || {};
+  const session = state.session;
+  return {
+    reportVersion: 3, generatedAtUnixMs: state.updatedAt,
+    software: { applicationVersion, implementationSha256, policyVersion: POLICY_VERSION, node: process.version,
+      dependencies: metrics.dependencies || null },
+    host: { logicalCpus: os.cpus().length, totalMemoryBytes: os.totalmem(), freeMemoryBytes: os.freemem(), loadAverage: os.loadavg() },
+    settings: state.limits,
+    recording: { paused: state.recording.paused, bytes: state.recording.bytes,
+      freeBytes: state.recording.freeBytes, pendingBytes: state.recording.pendingBytes,
+      droppedPending: state.recording.droppedPending,
+      sessions: state.recording.sessions.map((item) => ({ startedAtUnixMs: item.startedAt,
+        status: item.video, droppedEvents: item.droppedEvents, error: redact(item.error), failureDetails: item.failureDetails || null, stderr: redact(item.stderr), errorCategory: errorCategory(item.error) })) },
+    training: { status: training.status, steps: training.steps, examples: training.examples,
+      selection: training.selection || null, evaluatedSteps: training.evaluatedSteps ?? null,
+      loss: training.loss, losses: training.losses || null, publication: training.publication || null,
+      trainingAudit: metrics.trainingAudit || null,
+      evaluation: training.evaluation || null, dataset: metrics.dataset || training.dataset || null,
+      resources: { decodeSeconds: metrics.decodeSeconds ?? null, trainingSeconds: metrics.trainingSeconds ?? null,
+        evaluationSeconds: metrics.evaluationSeconds ?? null, probeSeconds: metrics.probeSeconds ?? null, beforeAuditSeconds: metrics.beforeAuditSeconds ?? null, checkpointSeconds: metrics.checkpointSeconds ?? null, cpuSeconds: metrics.cpuSeconds ?? null,
+        peakRssMiB: metrics.peakRssMiB ?? null },
+      history: training.history || [], rejectionReasons: (training.rejectionReasons || []).map(redact), rejectionCategories: (training.rejectionReasons || []).map(errorCategory),
+      error: redact(training.error), errorCategory: errorCategory(training.error), nextAttemptAtUnixMs: training.nextAttemptAt },
+    model: session ? { id: session.modelId, checkpointId: session.checkpointId, threshold: session.threshold, appliedThreshold: session.appliedThreshold } : null,
+    controller: controller ? { modelId: controller.modelId, sensorPresent: controller.sensorPresent || null,
+      stopReason: redact(controller.stopReason), stopCategory: errorCategory(controller.stopReason), history: (controller.history || []).map((item) => ({
+        ...item, submissions: item.submissions.map((submission) => ({ type: submission.type,
+          command: submission.command, result: redact(submission.result) })) })) } : null,
+    sensorFields: metrics.policy?.sensorFields || null,
+    timing: { alignment: 'server_receipt', browserVideoLatencyMs: null, wheelCommandExpiryMs: null },
+    events: state.activity.map(({ at, message, detail, error, stderr }) => ({ atUnixMs: at, message: redact(message), detail: redact(detail), error: error || null, stderr: redact(stderr) })),
+    unavailable: ['browser_video_latency', 'actual_physical_command_execution', 'live_training_cpu_utilization'],
+  };
+}
+module.exports = { report };

server/src/services/roverLearningService/driving.js

@@ -0,0 +1,203 @@
+// Autonomy owns a live user's control context, never an administrative identity.
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+const roverManager = require('../roverManager');
+const { isVerified, isDeterred } = require('../verificationService');
+const { isAdmin } = require('../roleService');
+const { submitCommand, issueCommand, subscribeCommandRecording } = require('../commandService');
+const { recordCommandRequest } = require('../commandService/recording');
+const { acquireModel } = require('./models');
+const { compatible } = require('./capabilities');
+
+const OPERATING = new Set(['drive', 'servo', 'headlight', 'raw']);
+const MODE_COMMANDS = new Set([128, 131, 132, 143].map((opcode) => Buffer.from([opcode]).toString('base64')));
+const clamp = (value, min, max) => Math.max(min, Math.min(max, Number(value) || 0));
+
+function canControl(socket, roverId) {
+  return Boolean(socket?.connected && isVerified(socket) && (isAdmin(socket) || !isDeterred(socket))
+    && roverManager.rovers.get(roverId)?.ws && roverManager.canDrive(roverId, socket));
+}
+
+function createDriving({ root, config, onChange, activity }) {
+  const sessions = new Map();
+  const recent = new Map();
+  let stopping = false;
+  let watchdog;
+  let unsubscribe;
+
+  function sendState(session, event) {
+    const input = session.worker.stdin;
+    if (input.destroyed || !input.writable || input.writableLength > 256 * 1024) return;
+    input.write(JSON.stringify(event) + '\n');
+  }
+
+  function transmit(session, command) {
+    if (session.status === 'stopping') return;
+    if (!canControl(session.socket, session.roverId)) { stop(session.roverId, 'Control permission lost'); return; }
+    if (!OPERATING.has(command?.type)) throw new Error('Unsupported model command');
+    if (command.type === 'raw' && !MODE_COMMANDS.has(command.raw)) throw new Error('Unsupported model operating-mode command');
+    const record = roverManager.rovers.get(session.roverId);
+    if (command.type === 'drive') {
+      const limit = Math.min(500, Number(record.meta?.maxWheelSpeed) || 500);
+      command.driveDirect = { left: Math.round(clamp(command.driveDirect?.left, -limit, limit)), right: Math.round(clamp(command.driveDirect?.right, -limit, limit)) };
+    }
+    if (command.type === 'servo') {
+      const servo = record.meta?.cameraServo;
+      if (!servo?.enabled) { session.submissions.push({ command, result: 'Servo unavailable' }); return; }
+      command.servo.angle = clamp(command.servo?.angle, servo.minAngle ?? 0, servo.maxAngle ?? 180);
+    }
+    const { type, ...data } = command;
+    recordCommandRequest(session.socket, { roverId: session.roverId, type, data }, (result) => {
+      session.lastResult = result.error || result.reason || 'issued';
+      session.submissions.push({ command, result: session.lastResult });
+      if (result.id) session.lastAction = { at: Date.now(), command };
+    }, (request, reply) => submitCommand(session.socket, request, reply), 'model');
+  }
+
+  function stop(roverId, reason = 'Stopped by user') {
+    const session = sessions.get(roverId);
+    if (!session) return Promise.resolve();
+    if (session.done) return session.done;
+    session.status = 'stopping';
+    session.stopReason = reason;
+    // A stop must still reach the rover after turn permission is revoked.
+    for (const command of [
+      { type: 'drive', driveDirect: { left: 0, right: 0 } },
+    ]) { try { issueCommand(roverId, command); } catch { /* Rover may already be offline. */ } }
+    session.worker.kill('SIGTERM');
+    const timer = setTimeout(() => session.worker.kill('SIGKILL'), 2000);
+    session.done = session.closed.then(() => {
+      clearTimeout(timer);
+      recent.delete(roverId);
+      recent.set(roverId, { modelId: session.modelId, stopReason: reason, history: session.history });
+      if (recent.size > 20) recent.delete(recent.keys().next().value);
+      sessions.delete(roverId);
+      activity('Controller stopped', { roverId, reason });
+      onChange();
+    });
+    onChange();
+    return session.done;
+  }
+
+  async function start(socket, roverId, modelId) {
+    if (stopping || !canControl(socket, roverId)) throw new Error('You cannot control this rover');
+    const acquired = await acquireModel(root, modelId);
+    const { directory, metadata } = acquired;
+    try {
+      if (!compatible(metadata, roverManager.rovers.get(roverId))) throw new Error('Model is not compatible with this rover');
+      if (stopping || !canControl(socket, roverId)) throw new Error('Control permission changed');
+      if (sessions.has(roverId)) throw new Error('Stop the current controller before starting another model');
+    } catch (error) { await acquired.release(); throw error; }
+    const worker = spawn(config.training.python, [path.join(__dirname, 'workers/infer.py'), '--model', directory,
+      '--url', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`, '--threads', String(config.driving.threads),
+      '--threshold', String(config.driving.actionThreshold)], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } });
+    const session = { worker, socket, roverId, modelId, checkpointId: metadata.id, modelName: modelId === 'latest' ? 'Latest' : modelId === 'previous' ? 'Previous' : metadata.name, status: 'starting', startedAt: Date.now(), lastPredictionAt: null, threshold: config.driving.actionThreshold, history: [] };
+    session.closed = new Promise((resolve) => worker.once('close', resolve));
+    session.closed.then(() => acquired.release()).catch((error) => activity('Checkpoint release failed', { error: error.message }));
+    sessions.set(roverId, session);
+    recent.delete(roverId);
+    let output = '';
+    let stderr = '';
+    worker.stdin.on('error', () => { stop(roverId, 'Worker input closed'); });
+    worker.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-1000); });
+    worker.on('error', (error) => { stop(roverId, error.message); });
+    worker.once('close', () => { stop(roverId, stderr || 'Worker exited'); });
+    worker.stdout.on('data', (chunk) => {
+      if (session.status === 'stopping') return;
+      output += chunk;
+      if (output.length > 1024 * 1024) { stop(roverId, 'Worker output exceeded limit'); return; }
+      let newline;
+      while ((newline = output.indexOf('\n')) >= 0) {
+        const line = output.slice(0, newline);
+        output = output.slice(newline + 1);
+        try {
+          const message = JSON.parse(line);
+          if (message.kind === 'error') throw new Error(message.message);
+          if (message.kind !== 'prediction') continue;
+          if (!Number.isFinite(message.frameAt) || Math.abs(Date.now() - message.frameAt) > config.driving.staleMs) throw new Error('Video is stale');
+          if (!Number.isFinite(message.sensorAt) || Math.abs(Date.now() - message.sensorAt) > config.driving.staleMs) throw new Error('Sensors are stale');
+          if (!canControl(socket, roverId)) throw new Error('Control permission lost');
+          session.status = 'running';
+          session.lastPredictionAt = Date.now();
+          session.frameAt = message.frameAt;
+          session.sensorAt = message.sensorAt;
+          session.latencyMs = message.latencyMs;
+          session.proposals = Array.isArray(message.proposals) ? message.proposals.slice(0, 64) : [];
+          session.appliedThreshold = message.threshold;
+          session.sensorPresent = message.sensorPresent;
+          session.submissions = [];
+          if (!Array.isArray(message.commands) || message.commands.length > 65) throw new Error('Invalid action batch');
+          session.predictionStatus = message.commands.length ? `${message.commands.length} command(s) proposed` : 'No command proposed';
+          // Manual commands intentionally do not stop or pause this controller.
+          for (const command of message.commands) transmit(session, command);
+          // Bound report history independently of camera and UI frame rates.
+          if (!session.history.length || Date.now() - session.history.at(-1).at >= 500) {
+            session.history.push({ at: Date.now(),
+              latencyMs: session.latencyMs, frameAgeMs: Date.now() - session.frameAt,
+              sensorAgeMs: Date.now() - session.sensorAt, threshold: session.appliedThreshold,
+              proposals: [...session.proposals].sort((a, b) => b.score - a.score).slice(0, 8).map(({ slot, score, reason, command }) => ({ slot, score, reason, command })),
+              submissions: session.submissions.slice(0, 16).map(({ command, result }) => ({ type: command.type, result, command })) });
+            if (session.history.length > 60) session.history.shift();
+          }
+        } catch (error) { stop(roverId, error.message); break; }
+      }
+    });
+    const sensors = roverManager.rovers.get(roverId)?.lastSensor?.decoded;
+    if (sensors) sendState(session, { kind: 'sensor', ts: Date.now(), data: sensors });
+    activity('Controller started', { roverId, model: metadata.name, socketId: socket.id });
+    onChange();
+  }
+
+  function sensor(event) {
+    const session = sessions.get(String(event.roverId));
+    if (session) sendState(session, { kind: 'sensor', ts: Date.now(), data: event.sensors });
+  }
+
+  function attach() {
+    roverManager.managerEvents.on('sensor', sensor);
+    unsubscribe = subscribeCommandRecording((event) => {
+      const session = sessions.get(String(event.roverId));
+      if (session && event.phase === 'dispatched') sendState(session, { kind: 'command', ts: event.ts, data: event.command });
+    });
+    watchdog = setInterval(() => {
+      for (const session of sessions.values()) {
+        if (!canControl(session.socket, session.roverId)) stop(session.roverId, 'Control permission lost');
+        else if (Date.now() - (session.lastPredictionAt || session.startedAt) > (session.lastPredictionAt ? config.driving.staleMs : 30000)) {
+          stop(session.roverId, 'No fresh model predictions');
+        }
+      }
+    }, 250);
+  }
+
+  return {
+    attach, start, stop, canControl,
+    setThreshold(socket, roverId, value) {
+      const session = sessions.get(roverId);
+      if (!session || session.status === 'stopping' || !canControl(socket, roverId)) throw new Error('You cannot adjust this controller');
+      if (typeof value !== 'number' || !Number.isFinite(value) || value < .05 || value > .99) throw new Error('Threshold must be between 0.05 and 0.99');
+      session.threshold = value;
+      sendState(session, { kind: 'settings', ts: Date.now(), threshold: value });
+    },
+    getDiagnostics(id) {
+      const session = sessions.get(id);
+      return session ? { modelId: session.modelId, sensorPresent: session.sensorPresent, history: session.history } : recent.get(id) || null;
+    },
+    isActive: (id) => sessions.has(id),
+    getState: (id) => {
+      const session = sessions.get(id);
+      if (!session) return null;
+      const { worker, socket, closed, done, history, ...state } = session;
+      return { ...state, startedBy: socket.data?.nickname || socket.id, socketId: socket.id };
+    },
+    shutdown: async () => {
+      stopping = true;
+      clearInterval(watchdog);
+      unsubscribe?.();
+      roverManager.managerEvents.off('sensor', sensor);
+      await Promise.all([...sessions.keys()].map((id) => stop(id, 'Service disabled')));
+    },
+    kill: () => { for (const session of sessions.values()) session.worker.kill('SIGKILL'); },
+  };
+}
+
+module.exports = { createDriving, canControl };

server/src/services/roverLearningService/errors.js

@@ -0,0 +1,18 @@
+// Keep actionable failures while removing credentials and user identifiers.
+function redact(value) {
+  if (value == null) return null;
+  return String(value).slice(0, 12000)
+    .replace(/\b(?:https?|rtsp|rtsps):\/\/[^\s'"<>]+/gi, '[redacted URL]')
+    .replace(/\b(Bearer\s+)\S+/gi, '$1[redacted]')
+    .replace(/((?:password|token|secret|authorization|cookie|api[_-]?key)\s*[=:]\s*)[^\s,;]+/gi, '$1[redacted]')
+    .replace(/\bcu_[a-z0-9]+/gi, '[redacted user]')
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, '[redacted email]')
+    .replace(/\/(?:home|Users)\/[^/\s]+/g, '/[user]');
+}
+function errorDetails(error, operation) {
+  return { atUnixMs: Date.now(), operation: redact(operation),
+    name: redact(error?.name), message: redact(error?.message ?? error),
+    code: redact(error?.code), syscall: redact(error?.syscall),
+    path: redact(error?.path), stack: redact(error?.stack) };
+}
+module.exports = { redact, errorDetails };

server/src/services/roverLearningService/index.js

@@ -0,0 +1,39 @@
+// The only resident part while disabled is this configuration lifecycle bridge.
+const { loadConfig, registerConfigurationHandler } = require('../../configuration');
+const logger = require('../../globals/logger').child('roverLearning');
+
+let runtime = null;
+let transitions = Promise.resolve();
+let shuttingDown = false;
+
+function apply(config) {
+  const transition = transitions.then(async () => {
+    if (runtime) {
+      const previous = runtime;
+      runtime = null;
+      await previous.stop();
+    }
+    if (!config?.enabled || shuttingDown) return;
+    const { createRuntime } = require('./runtime');
+    const next = createRuntime({ config, logger });
+    try {
+      await next.start();
+      runtime = next;
+    } catch (error) {
+      await next.stop();
+      throw error;
+    }
+  });
+  transitions = transition.catch(() => undefined);
+  return transition;
+}
+
+registerConfigurationHandler('roverLearning', apply);
+apply(loadConfig().roverLearning).catch((error) => {
+  logger.error('Unable to start rover learning recording', { error: error.message });
+});
+
+// Signal handlers are installed by the enabled runtime, not by this bridge.
+module.exports = {
+  stop: () => { shuttingDown = true; return apply({ enabled: false }); },
+};

server/src/services/roverLearningService/models.js

@@ -0,0 +1,173 @@
+const { POLICY_VERSION } = require('./capabilities');
+// Mutable aliases reference immutable checkpoints. All readers and retention share
+// a queue so a model cannot disappear between resolving its alias and loading it.
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { randomUUID, randomInt } = require('node:crypto');
+const UUID = /^[0-9a-f-]{36}$/;
+let queue = Promise.resolve();
+const leases = new Map();
+const retired = new Set();
+function serial(task) {
+  const result = queue.then(task);
+  queue = result.catch(() => {});
+  return result;
+}
+const checkpointRoot = (root) => path.join(root, 'training', 'checkpoints');
+async function index(root) {
+  try { return JSON.parse(await fs.readFile(path.join(checkpointRoot(root), 'index.json'), 'utf8')); }
+  catch (error) { if (error.code === 'ENOENT') return {}; throw error; }
+}
+async function metadata(directory) {
+  const model = JSON.parse(await fs.readFile(path.join(directory, 'model.json'), 'utf8'));
+  if (model?.id !== path.basename(directory)) throw new Error('Model identity mismatch');
+  return model;
+}
+async function prune(root, state) {
+  const directory = checkpointRoot(root);
+  await fs.mkdir(directory, { recursive: true });
+  for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    if (/^\.[0-9a-f-]{36}\.pending$/.test(entry.name)) {
+      await fs.rm(path.join(directory, entry.name), { recursive: true });
+      continue;
+    }
+    if (!UUID.test(entry.name)) continue;
+    const target = path.join(directory, entry.name);
+    if (!Object.values(state).includes(entry.name)) {
+      if (leases.has(target)) retired.add(target);
+      else { await fs.rm(target, { recursive: true }); retired.delete(target); }
+    }
+  }
+}
+function adoptCheckpoint(root, model, promote) {
+  return serial(async () => {
+    const state = await index(root);
+    if (!UUID.test(model.id)) throw new Error('Invalid checkpoint identity');
+    await metadata(path.join(checkpointRoot(root), model.id));
+    state.candidate = model.id;
+    if (promote) { state.previous = state.latest || null; state.latest = model.id; }
+    const filename = path.join(checkpointRoot(root), 'index.json');
+    await fs.writeFile(`${filename}.tmp`, JSON.stringify(state));
+    await fs.rename(`${filename}.tmp`, filename);
+    await prune(root, state);
+  });
+}
+function acquireModel(root, id) {
+  return serial(async () => {
+    const state = await index(root);
+    const alias = ['latest', 'previous'].includes(id);
+    const resolved = alias ? state[id] : id;
+    if (!UUID.test(resolved || '')) throw new Error('Model is not available');
+    const directory = path.join(alias ? checkpointRoot(root) : path.join(root, 'models'), resolved);
+    const model = await metadata(directory);
+    leases.set(directory, (leases.get(directory) || 0) + 1);
+    let released = false;
+    return { directory, metadata: model, release: () => serial(async () => {
+      if (released) return;
+      released = true;
+      const count = leases.get(directory) - 1;
+      if (count) leases.set(directory, count); else leases.delete(directory);
+      // Delete only a specific checkpoint already retired by the trainer, not
+      // a worker's new export awaiting adoption.
+      if (!count && retired.has(directory)) {
+        await fs.rm(directory, { recursive: true, force: true });
+        retired.delete(directory);
+      }
+    }) };
+  });
+}
+function saveSnapshot(root) {
+  return serial(async () => {
+    const state = await index(root);
+    if (!UUID.test(state.candidate || '')) throw new Error('No completed learner checkpoint yet');
+    const source = path.join(checkpointRoot(root), state.candidate);
+    const original = await metadata(source);
+    if (original.specification?.version !== POLICY_VERSION) throw new Error('Waiting for a completed command-policy checkpoint');
+    const id = randomUUID();
+    const words = [['amber', 'silver', 'quiet', 'curious'], ['otter', 'finch', 'fox', 'wren']];
+    const name = `${words[0][randomInt(4)]}-${words[1][randomInt(4)]}-${id.slice(0, 8)}`;
+    const model = { ...original, id, name, createdAt: Date.now(), sourceCheckpointId: original.id,
+      snapshot: true, experimental: original.evaluation !== 'passed imitation gate' };
+    const destination = path.join(root, 'models', id);
+    const pending = path.join(root, 'models', `.${id}.pending`);
+    await fs.mkdir(pending, { recursive: true });
+    try {
+      await fs.copyFile(path.join(source, 'weights.pt'), path.join(pending, 'weights.pt'));
+      await fs.writeFile(path.join(pending, 'model.json'), JSON.stringify(model));
+      await fs.rename(pending, destination);
+    } catch (error) { await fs.rm(pending, { recursive: true, force: true }); throw error; }
+    return model;
+  });
+}
+// Retire recognized older policies before workers start. Unknown/malformed or
+// future manifests are not evidence that a directory is disposable.
+function removeObsoleteModels(root) {
+  return serial(async () => {
+    const obsolete = [];
+    for (const directory of [path.join(root, 'models'), checkpointRoot(root)]) {
+      let entries;
+      try { entries = await fs.readdir(directory, { withFileTypes: true }); }
+      catch (error) { if (error.code === 'ENOENT') continue; throw error; }
+      for (const entry of entries) {
+        if (!entry.isDirectory() || !UUID.test(entry.name)) continue;
+        const target = path.join(directory, entry.name);
+        let model;
+        try { model = await metadata(target); }
+        catch (error) {
+          if (error.code === 'ENOENT' || error instanceof SyntaxError || error.message === 'Model identity mismatch') continue;
+          throw error;
+        }
+        const version = model.specification?.version;
+        if (Number.isInteger(version) && version >= 1 && version < POLICY_VERSION) {
+          obsolete.push({ target, checkpoint: directory === checkpointRoot(root), id: entry.name });
+        }
+      }
+    }
+    const state = await index(root);
+    const oldCheckpoints = new Set(obsolete.filter((entry) => entry.checkpoint).map((entry) => entry.id));
+    let changed = false;
+    for (const alias of ['candidate', 'latest', 'previous']) {
+      if (oldCheckpoints.has(state[alias])) { delete state[alias]; changed = true; }
+    }
+    // Remove aliases atomically first, so interruption cannot leave the catalog
+    // referring to a checkpoint that cleanup already deleted.
+    if (changed) {
+      const filename = path.join(checkpointRoot(root), 'index.json');
+      await fs.writeFile(`${filename}.tmp`, JSON.stringify(state));
+      await fs.rename(`${filename}.tmp`, filename);
+    }
+    for (const { target } of obsolete) {
+      if (leases.has(target)) retired.add(target);
+      else { await fs.rm(target, { recursive: true, force: true }); retired.delete(target); }
+    }
+    // These filenames belong to superseded optimizer/scheduler lineages, not
+    // policy versions. The current command policy uses lineage v6.
+    for (const suffix of ['', '-v1', '-v2', '-v3', '-v4', '-v5']) {
+      for (const filename of [`resume${suffix}.pt`, `resume${suffix}.tmp`,
+        `schedule${suffix}.json`, `schedule${suffix}.json.tmp`]) {
+        await fs.rm(path.join(root, 'training', filename), { force: true });
+      }
+    }
+  });
+}
+
+function listModels(root) {
+  return serial(async () => {
+    let entries;
+    try { entries = await fs.readdir(path.join(root, 'models'), { withFileTypes: true }); }
+    catch (error) { if (error.code !== 'ENOENT') throw error; entries = []; }
+    const models = [];
+    const state = await index(root);
+    for (const alias of ['latest', 'previous']) {
+      if (!state[alias]) continue;
+      const model = await metadata(path.join(checkpointRoot(root), state[alias]));
+      models.push({ ...model, id: alias, checkpointId: model.id, name: alias === 'latest' ? 'Latest' : 'Previous', automatic: true });
+    }
+    for (const entry of entries) {
+      if (entry.isDirectory() && UUID.test(entry.name)) models.push(await metadata(path.join(root, 'models', entry.name)));
+    }
+    return models;
+  });
+}
+module.exports = { removeObsoleteModels, listModels, acquireModel, adoptCheckpoint, saveSnapshot };

server/src/services/roverLearningService/recording.js

@@ -0,0 +1,185 @@
+// One bounded recording session. Video packets retain receiver wall-clock PTS;
+// telemetry uses the same server clock, not an assumed browser playback clock.
+const { errorDetails, redact } = require('./errors');
+const fs = require('node:fs');
+const fsp = require('node:fs/promises');
+const path = require('node:path');
+const { randomUUID } = require('node:crypto');
+const { spawn } = require('node:child_process');
+const { controlProfile } = require('./capabilities');
+
+const MAX_BUFFER_BYTES = 2 * 1024 * 1024;
+
+async function saveMetadata(directory, metadata) {
+  const filename = path.join(directory, 'session.json');
+  await fsp.writeFile(`${filename}.tmp`, JSON.stringify(metadata, null, 2));
+  await fsp.rename(`${filename}.tmp`, filename);
+}
+
+async function createRecording({ root, roverId, snapshot, logger, docked = false }) {
+  const id = `${Date.now()}-${randomUUID()}`;
+  const directory = path.join(root, id);
+  await fsp.mkdir(directory);
+  const metadata = {
+    version: 1, id, roverId, startedAt: Date.now(), endedAt: null,
+    controlProfile: controlProfile(snapshot.meta), wheelSpeedLimit: snapshot.meta?.maxWheelSpeed ?? 500,
+    timing: 'Video PTS uses server receiver Unix seconds. Events use Unix milliseconds and process monotonic nanoseconds. Browser display delay is not measured.',
+    video: { file: 'video.mkv', codec: 'source-copy', timestampUnit: 'seconds', origin: 'unix' },
+    events: 'events.ndjson', droppedEvents: 0, bufferOnly: docked,
+    trainingStartedAt: docked ? null : Date.now(),
+  };
+  await saveMetadata(directory, metadata);
+  const output = fs.createWriteStream(path.join(directory, 'events.ndjson'), { flags: 'wx' });
+  let buffered = [];
+  let bufferedBytes = 0;
+  let dockedAt = null;
+  let pruning = Promise.resolve();
+  let failure = null;
+  let failureDetails = null;
+  let stopping = false;
+  let stopPromise;
+  let child;
+  let childClosed = false;
+  let stderr = '';
+  let videoResult = null;
+  let resolveVideo;
+  const videoClosed = new Promise((resolve) => { resolveVideo = resolve; });
+  output.on('error', (error) => {
+    failure = error.message; failureDetails = errorDetails(error, 'recording session');
+    // Stop file growth immediately when telemetry is no longer writable.
+    child?.kill('SIGTERM');
+  });
+
+  function append(kind, data, timestamp = {}) {
+    if (stopping || failure) return false;
+    try {
+      const line = JSON.stringify({
+        ts: Date.now(), monotonicNs: process.hrtime.bigint().toString(), ...timestamp, kind, data,
+      }) + '\n';
+      if (metadata.bufferOnly) {
+        const now = Date.now();
+        buffered.push({ at: now, line });
+        bufferedBytes += Buffer.byteLength(line);
+        while (buffered.length && buffered[0].at < now - 5000) {
+          bufferedBytes -= Buffer.byteLength(buffered.shift().line);
+        }
+        while (bufferedBytes > MAX_BUFFER_BYTES && buffered.length) {
+          metadata.droppedEvents += 1;
+          bufferedBytes -= Buffer.byteLength(buffered.shift().line);
+        }
+        return true;
+      }
+      // Never wait for disk from a control/sensor callback. Explicit loss counts
+      // prevent an overloaded recorder from pretending its history is complete.
+      if (output.writableLength + Buffer.byteLength(line) > MAX_BUFFER_BYTES) {
+        metadata.droppedEvents += 1;
+        return true;
+      }
+      output.write(line);
+    } catch (error) {
+      metadata.droppedEvents += 1;
+      failure = error.message; failureDetails = errorDetails(error, 'recording session');
+      child?.kill('SIGTERM');
+    }
+    return true;
+  }
+
+  append('snapshot', snapshot);
+  // Read the local published stream, as replay does, without depending on the
+  // replay service or re-encoding camera video. No audio/chat is captured.
+  child = spawn(process.env.FFMPEG_BIN || 'ffmpeg', [
+    '-hide_banner', '-loglevel', 'warning', '-nostdin', '-n',
+    '-rtsp_transport', 'tcp', '-timeout', '10000000',
+    '-use_wallclock_as_timestamps', '1', '-copyts',
+    '-i', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`,
+    '-map', '0:v:0', '-an', '-c:v', 'copy', '-avoid_negative_ts', 'disabled',
+    // Keyframe-aligned chunks keep a disposable dock buffer without repeatedly
+    // reconnecting to RTSP. Extra edge packets are filtered by trainingStartedAt.
+    '-f', 'segment', '-segment_time', '2', '-segment_format', 'matroska',
+    '-reset_timestamps', '0', path.join(directory, 'video-%09d.mkv'),
+  ], { stdio: ['ignore', 'ignore', 'pipe'] });
+  child.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString()).slice(-4096); });
+  child.on('error', (error) => { failure = error.message; failureDetails = errorDetails(error, 'recording session'); });
+  child.once('close', (code, signal) => {
+    childClosed = true;
+    videoResult = { code, signal, stderr };
+    if (!stopping) failure ||= `Video recorder exited (${code ?? signal})`;
+    resolveVideo();
+  });
+
+  function updateDock(isDocked) {
+    if (stopping) return;
+    if (!isDocked) {
+      dockedAt = null;
+      if (metadata.bufferOnly) {
+        metadata.bufferOnly = false;
+        metadata.trainingStartedAt = Date.now() - 5000;
+        for (const item of buffered) output.write(item.line);
+        buffered = [];
+        bufferedBytes = 0;
+      }
+    } else if (!metadata.bufferOnly && dockedAt == null) dockedAt = Date.now();
+  }
+
+  function pruneBuffer() {
+    pruning = pruning.then(async () => {
+      if (!metadata.bufferOnly || stopping) return;
+      const files = (await fsp.readdir(directory)).filter((name) => /^video-\d+\.mkv$/.test(name)).sort();
+      // Never unlink the open segment; retain five seconds plus the preceding
+      // keyframe segment so the decoder can recover the full lead-in.
+      for (const name of files.slice(0, -2)) {
+        const filename = path.join(directory, name);
+        const stat = await fsp.stat(filename);
+        if (!metadata.bufferOnly || stopping) return;
+        if (stat.mtimeMs < Date.now() - 5000) await fsp.unlink(filename);
+      }
+    }).catch((error) => { failure = error.message; failureDetails = errorDetails(error, 'recording session'); child.kill('SIGTERM'); });
+  }
+
+  async function stop(reason) {
+    if (stopPromise) return stopPromise;
+    stopPromise = (async () => {
+      append('sessionEnd', { reason, droppedEvents: metadata.droppedEvents });
+      stopping = true;
+      if (!childClosed) child.kill('SIGTERM');
+      // A stuck RTSP read must not leave a recorder behind after disable.
+      const killTimer = setTimeout(() => { if (!childClosed) child.kill('SIGKILL'); }, 2000);
+      await videoClosed;
+      await pruning;
+      clearTimeout(killTimer);
+      await new Promise((resolve) => {
+        if (output.closed) return resolve();
+        output.once('close', resolve);
+        output.end();
+      });
+      if (metadata.bufferOnly) {
+        await fsp.rm(directory, { recursive: true, force: true });
+        return;
+      }
+      metadata.video.files = (await fsp.readdir(directory)).filter((name) => /^video-\d+\.mkv$/.test(name)).sort();
+      metadata.endedAt = Date.now();
+      metadata.reason = reason;
+      metadata.error = failure;
+      metadata.failureDetails = failureDetails;
+      metadata.video.result = videoResult;
+      await saveMetadata(directory, metadata);
+      if (failure) logger.warn('Recording session incomplete', { roverId, id, error: failure, failure: failureDetails, stderr });
+    })();
+    return stopPromise;
+  }
+
+  return {
+    id, directory, startedAt: metadata.startedAt, append, stop, updateDock, pruneBuffer,
+    get buffered() { return metadata.bufferOnly; },
+    get dockTailComplete() { return dockedAt != null && Date.now() - dockedAt >= 5000; },
+    get trainingStartedAt() { return metadata.trainingStartedAt; },
+    get failed() { return Boolean(failure); },
+    get stopping() { return stopping; },
+    getStatus: () => ({ id, roverId, startedAt: metadata.startedAt, droppedEvents: metadata.droppedEvents,
+      video: childClosed ? 'stopped' : metadata.bufferOnly ? 'Dock lead-in buffer' : 'Recording human driving', error: redact(failure), failureDetails, stderr: redact(stderr), stopping }),
+    // Process exit is synchronous; normal disable uses stop() and awaits close.
+    kill: () => { if (!childClosed) child.kill('SIGKILL'); },
+  };
+}
+
+module.exports = { createRecording, saveMetadata };

server/src/services/roverLearningService/runtime.js

@@ -0,0 +1,316 @@
+// Enabled-only recording, training, controller, and card lifecycle.
+const { redact, errorDetails } = require('./errors');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { resolveDataPath } = require('../../helpers/dataPaths');
+const roverManager = require('../roverManager');
+const io = require('../../globals/io');
+const { turnEvents } = require('../turnService');
+const { modeEvents } = require('../modeManager');
+const { isAdmin, roleEvents } = require('../roleService');
+const { isDeterred, verificationEvents } = require('../verificationService');
+const { subscribeCommandRecording, commandEvents } = require('../commandService');
+const { createRecording } = require('./recording');
+const { recoverRecordings, enforceRetention } = require('./storage');
+const { createTrainer } = require('./training');
+const { createDriving } = require('./driving');
+const { removeObsoleteModels } = require('./models');
+const { createGateway } = require('./socketGateway');
+
+function createRuntime({ config, logger }) {
+  const root = resolveDataPath('rover-learning');
+  const recordingsRoot = path.join(root, 'recordings');
+  const recentActivity = [];
+  function activity(message, details = {}) {
+    recentActivity.push({ at: Date.now(), message, roverId: details.roverId || null,
+      detail: redact(details.reason || details.error || details.model),
+      error: details.failure || (details.error ? errorDetails(details.error, message) : null),
+      stderr: redact(details.stderr) });
+    if (recentActivity.length > 40) recentActivity.shift();
+  }
+  const baseLogger = logger;
+  logger = Object.fromEntries(['info', 'warn', 'error'].map((level) => [level, (message, details) => {
+    baseLogger[level](message, details); activity(message, details);
+  }]));
+  const trainer = createTrainer({ root, config: config.training, actionThreshold: config.driving.actionThreshold, logger });
+  const driving = createDriving({ root, config, onChange: refreshEligibility, activity });
+  let gateway;
+  let dockTimer;
+  let diskState = { bytes: null, free: null };
+  const sessions = new Map();
+  const unsubscribers = [];
+  let running = false;
+  let timer;
+  let maintenance = Promise.resolve();
+  let stopPromise;
+  let diskPaused = false;
+  let busy = false;
+  let wakeRequested = false;
+  let pending = [];
+  let pendingBytes = 0;
+  let droppedPending = 0;
+  let eligibleRovers = new Set();
+
+  function hasHumanController(roverId) {
+    // Even manual inputs during autonomy remain excluded: model-influenced
+    // video must not silently become human demonstration training data.
+    if (driving.isActive(roverId)) return false;
+    const record = roverManager.rovers.get(roverId);
+    if (!record?.ws) return false;
+    // Admin permission alone must not record the whole fleet. Require actual
+    // driver membership, a live browser socket, and current control eligibility.
+    for (const socketId of record.drivers || []) {
+      const socket = io.sockets.sockets.get(socketId);
+      if (socket?.connected && (isAdmin(socket) || !isDeterred(socket))
+          && roverManager.canDrive(roverId, socket)) return true;
+    }
+    return false;
+  }
+
+  function currentEligibleRovers() {
+    return new Set([...roverManager.rovers.keys()].filter(hasHumanController));
+  }
+
+  function refreshEligibility() {
+    const next = currentEligibleRovers();
+    const changed = next.size !== eligibleRovers.size || [...next].some((id) => !eligibleRovers.has(id));
+    eligibleRovers = next;
+    if (!changed) return;
+    // Stop the stream immediately even if disk maintenance is in progress.
+    // The serialized maintenance pass still owns closing/removing the session.
+    for (const [id, session] of sessions) {
+      if (!next.has(id)) session.stop('no-human-controller').catch((error) => {
+        logger.warn('Unable to finalize unattended recording', { roverId: id, error: error.message, failure: errorDetails(error, 'rover learning runtime') });
+      });
+    }
+    schedule();
+  }
+
+  function append(kind, event) {
+    if (!running || event?.roverId == null) return;
+    const roverId = String(event.roverId);
+    if (!hasHumanController(roverId)) return;
+    const timestamp = { ts: Date.now(), monotonicNs: process.hrtime.bigint().toString() };
+    if (sessions.get(roverId)?.append(kind, event, timestamp)) return;
+    if (diskPaused || !roverManager.rovers.has(roverId)) return;
+    // Session creation/rotation is asynchronous. Keep a bounded snapshot of
+    // events arriving during that gap, with their original observation times.
+    try {
+      const serialized = JSON.stringify(event);
+      const bytes = Buffer.byteLength(serialized);
+      if (pendingBytes + bytes > 4 * 1024 * 1024) {
+        droppedPending += 1;
+        return;
+      }
+      pending.push({ roverId, kind, serialized, timestamp, bytes });
+      pendingBytes += bytes;
+    } catch { droppedPending += 1; }
+  }
+
+  function listen(emitter, name, handler) {
+    emitter.on(name, handler);
+    unsubscribers.push(() => emitter.off(name, handler));
+  }
+
+  function snapshot(record) {
+    // Do not serialize the live record: it contains sockets and circular state.
+    return {
+      meta: record.meta, sensors: record.lastSensor, hostStats: record.lastHostStats,
+      batteryState: record.batteryState, headlightState: record.headlightState,
+      laserState: record.laserState, drivers: [...(record.drivers || [])],
+      locked: record.locked, privateOpen: record.privateOpen,
+    };
+  }
+
+  async function closeSession(roverId, reason) {
+    const session = sessions.get(roverId);
+    if (!session) return;
+    try {
+      await session.stop(reason);
+    } finally {
+      // Keep closing children reachable by the synchronous process-exit hook.
+      sessions.delete(roverId);
+    }
+  }
+
+  async function maintain() {
+    if (!running) return;
+    pending = pending.filter((event) => hasHumanController(event.roverId));
+    pendingBytes = pending.reduce((sum, event) => sum + event.bytes, 0);
+    for (const [roverId, session] of sessions) {
+      const record = roverManager.rovers.get(roverId);
+      if (!hasHumanController(roverId) || session.stopping || session.failed || session.dockTailComplete || (!session.buffered && Date.now() - session.trainingStartedAt >= config.recording.sessionSeconds * 1000)) {
+        await closeSession(roverId, !record?.ws ? 'offline' : !hasHumanController(roverId) ? 'no-human-controller' : session.failed ? 'recorder-failed' : session.dockTailComplete ? 'docked' : 'rotation');
+      }
+    }
+    if (!running) return;
+    const retention = await enforceRetention(
+      recordingsRoot, new Set([...sessions.values()].map((session) => session.id).concat([...trainer.leases])),
+      config.recording.maxGiB * 1024 ** 3, config.recording.minimumFreeGiB * 1024 ** 3,
+    );
+    diskState = retention;
+    if (retention.paused) {
+      if (!diskPaused) logger.warn('Recording paused for disk budget', retention);
+      diskPaused = true;
+      pending = [];
+      pendingBytes = 0;
+      await trainer.cancel();
+      // Close active files to make them eligible for pruning on the next tick.
+      await Promise.all([...sessions.keys()].map((id) => closeSession(id, 'disk-budget')));
+      return;
+    }
+    if (diskPaused) logger.info('Recording resumed after disk budget recovery');
+    diskPaused = false;
+    for (const [key, record] of roverManager.rovers) {
+      if (!running) break;
+      const roverId = String(key);
+      if (!hasHumanController(roverId) || sessions.has(roverId)) continue;
+      const session = await createRecording({ root: recordingsRoot, roverId, snapshot: snapshot(record), logger, docked: Boolean(record.lastSensor?.decoded?.chargingSources?.homeBase) });
+      sessions.set(roverId, session);
+      // Filesystem awaits can span a disconnect or turn change.
+      if (!running || !hasHumanController(roverId)) {
+        await closeSession(roverId, running ? 'no-human-controller' : 'disabled');
+        continue;
+      }
+      if (droppedPending) {
+        session.append('captureGap', { droppedEventsAcrossRovers: droppedPending });
+        droppedPending = 0;
+      }
+      pending = pending.filter((event) => {
+        if (event.roverId !== roverId) return true;
+        session.append(event.kind, JSON.parse(event.serialized), event.timestamp);
+        pendingBytes -= event.bytes;
+        return false;
+      });
+    }
+    if (running) {
+      try {
+        await trainer.tick(new Set([...sessions.values()].map((session) => session.id)));
+      } catch (error) {
+        // Training availability must not tear down otherwise healthy capture.
+        logger.error('Unable to schedule training', { error: error.message, failure: errorDetails(error, 'rover learning runtime') });
+      }
+    }
+  }
+
+  function schedule() {
+    if (!running) return;
+    if (busy) { wakeRequested = true; return; }
+    clearTimeout(timer);
+    busy = true;
+    // The next tick is scheduled after completion: disk slowness cannot build
+    // an unbounded backlog of retention tasks or overlapping recorder starts.
+    maintenance = maintain().catch(async (error) => {
+      logger.error('Recording maintenance failed; closing recorders', { error: error.message, failure: errorDetails(error, 'recording maintenance') });
+      await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'storage-error')));
+    }).finally(() => {
+      busy = false;
+      if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
+      wakeRequested = false;
+    });
+  }
+
+  function killChildren() {
+    driving.kill();
+    trainer.kill();
+    for (const session of sessions.values()) session.kill();
+  }
+  function onSignal() {
+    // Other server services can exit the process before asynchronous cleanup
+    // finishes. Signal children immediately, and keep an exit fallback too.
+    for (const session of sessions.values()) session.stop('server-shutdown').catch(() => undefined);
+    stop().catch((error) => logger.warn('Recording shutdown failed', { error: error.message, failure: errorDetails(error, 'rover learning runtime') }));
+  }
+
+  async function start() {
+    await fs.mkdir(recordingsRoot, { recursive: true });
+    await recoverRecordings(recordingsRoot);
+    await removeObsoleteModels(root);
+    await trainer.start();
+    running = true;
+    driving.attach();
+    dockTimer = setInterval(() => {
+      for (const [id, session] of sessions) {
+        session.updateDock(Boolean(roverManager.rovers.get(id)?.lastSensor?.decoded?.chargingSources?.homeBase));
+        session.pruneBuffer();
+        if (session.dockTailComplete) schedule();
+      }
+    }, 250);
+    gateway = createGateway({ root, driving, activity, getSystemState: () => ({
+      recording: { paused: diskPaused, bytes: diskState.bytes, freeBytes: diskState.free,
+        pendingBytes, droppedPending, sessions: [...sessions.values()].map((session) => session.getStatus()) },
+      training: trainer.getState(),
+      limits: { recording: config.recording, training: { ...config.training, python: undefined }, driving: config.driving },
+      activity: [...recentActivity].reverse(),
+    }) });
+    eligibleRovers = currentEligibleRovers();
+    unsubscribers.push(subscribeCommandRecording((event) => append('command', event)));
+    // ACK analytics also provide latency/type context; detailed acknowledgements
+    // arrive through the recording hook and share the command ID.
+    listen(commandEvents, 'observation', (event) => append('commandOutcome', event));
+    listen(roverManager.managerEvents, 'sensor', (event) => append('sensor', {
+      ...event,
+      // The manager stores the exact frame before emitting this synchronous
+      // event, so raw capture needs no extra hook in rover sensor processing.
+      frame: roverManager.rovers.get(event.roverId)?.lastSensor?.raw,
+    }));
+    for (const kind of ['hostStats', 'driver', 'switch', 'lock', 'private', 'privateSafety', 'help', 'dockGuard']) {
+      listen(roverManager.managerEvents, kind, (event) => {
+        append(kind, event);
+        if (['driver', 'switch', 'lock', 'private', 'privateSafety'].includes(kind)) refreshEligibility();
+      });
+    }
+    for (const kind of ['activeDriver', 'queue']) {
+      listen(turnEvents, kind, (event) => { append(`turn:${kind}`, event); refreshEligibility(); });
+    }
+    listen(modeEvents, 'change', refreshEligibility);
+    listen(roleEvents, 'change', refreshEligibility);
+    listen(verificationEvents, 'change', refreshEligibility);
+    listen(roverManager.managerEvents, 'rover', (event) => {
+      append('rover', { roverId: event.roverId, action: event.action, snapshot: event.record ? snapshot(event.record) : null });
+      if (event.action === 'upsert' || event.action === 'removed') refreshEligibility();
+    });
+    process.on('SIGTERM', onSignal);
+    process.on('SIGINT', onSignal);
+    process.on('exit', killChildren);
+    busy = true;
+    maintenance = maintain();
+    try {
+      await maintenance;
+    } finally {
+      busy = false;
+    }
+    if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
+    wakeRequested = false;
+    logger.info('Rover learning enabled', { root, training: config.training.enabled });
+  }
+
+  async function stop() {
+    if (stopPromise) return stopPromise;
+    running = false;
+    gateway?.close();
+    clearTimeout(timer);
+    clearInterval(dockTimer);
+    for (const unsubscribe of unsubscribers.splice(0)) unsubscribe();
+    stopPromise = (async () => {
+      const trainingStop = trainer.stop();
+      const drivingStop = driving.shutdown();
+      await maintenance.catch(() => undefined);
+      const results = await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'disabled')));
+      await trainingStop;
+      await drivingStop;
+      process.off('SIGTERM', onSignal);
+      process.off('SIGINT', onSignal);
+      process.off('exit', killChildren);
+      pending = [];
+      pendingBytes = 0;
+      const failed = results.find((result) => result.status === 'rejected');
+      if (failed) throw failed.reason;
+    })();
+    return stopPromise;
+  }
+
+  return { start, stop, getTrainingState: trainer.getState };
+}
+
+module.exports = { createRuntime };

server/src/services/roverLearningService/socketGateway.js

@@ -0,0 +1,127 @@
+// Complete card snapshots; live diagnostics opt into 10 Hz, ordinary cards use 1 Hz.
+const io = require('../../globals/io');
+const roverManager = require('../roverManager');
+const { isVerified } = require('../verificationService');
+const { isAdmin } = require('../roleService');
+const { listModels, saveSnapshot } = require('./models');
+const { report } = require('./diagnostics');
+const { compatible } = require('./capabilities');
+
+function createGateway({ root, driving, getSystemState, activity }) {
+  const bindings = new Map();
+  const viewers = new Map();
+  let models = [];
+  let modelError = null;
+  let lastCatalog = 0;
+  let closed = false;
+  let refreshing = false;
+  let lastOrdinary = 0;
+
+  function stateFor(socket, roverId) {
+    if (!isVerified(socket)) return { available: false };
+    const visible = roverId && roverManager.canSeeRover(roverId, socket);
+    const id = visible ? roverId : null;
+    const session = id ? driving.getState(id) : null;
+    const canControl = Boolean(id && driving.canControl(socket, id));
+    const system = getSystemState();
+    const state = {
+      available: true, roverId: id, updatedAt: Date.now(),
+      control: { canSnapshot: isAdmin(socket), canStart: canControl && !session, canAdjust: canControl && Boolean(session), canStop: Boolean(session && (canControl || session.socketId === socket.id)),
+        reason: !id ? 'Select a rover first' : !canControl ? 'You do not currently have control of this rover' : null },
+      session,
+      models: models.filter((model) => model.trainedRovers?.some((rover) => roverManager.canSeeRover(rover, socket)))
+        .map((model) => ({ id: model.id, checkpointId: model.checkpointId || model.id, automatic: Boolean(model.automatic), experimental: Boolean(model.experimental), name: model.name, createdAt: model.createdAt,
+          trainingSteps: model.trainingSteps, examplesProcessed: model.examplesProcessed,
+          weightBytes: model.weightBytes, parameters: model.parameters, trainingLoss: model.trainingLoss,
+          formatVersion: model.specification?.version, evaluation: model.evaluation, validation: model.validation, distinctMinutes: model.distinctMinutes, controls: model.specification?.commands?.length || 0,
+          compatible: Boolean(id && compatible(model, roverManager.rovers.get(id))) })),
+      catalogError: modelError,
+      recording: { ...system.recording, sessions: system.recording.sessions.filter((s) => roverManager.canSeeRover(s.roverId, socket)) },
+      training: { ...system.training, sessions: undefined },
+      limits: system.limits,
+      activity: system.activity.filter((entry) => !entry.roverId || roverManager.canSeeRover(entry.roverId, socket)),
+    };
+    state.diagnostics = viewers.get(socket)?.live ? report(state, id ? driving.getDiagnostics(id) : null) : null;
+    return state;
+  }
+
+  async function refresh() {
+    if (closed || refreshing) return;
+    refreshing = true;
+    try {
+      if (Date.now() - lastCatalog >= 15000) {
+        lastCatalog = Date.now();
+        try { models = await listModels(root); modelError = null; }
+        catch (error) { modelError = error.message; }
+      }
+      if (closed) return;
+      const ordinary = Date.now() - lastOrdinary >= 1000;
+      if (ordinary) lastOrdinary = Date.now();
+      for (const [socket, view] of viewers) {
+        if (ordinary || view.live || driving.isActive(view.roverId)) socket.volatile.emit('roverLearning:state', stateFor(socket, view.roverId));
+      }
+    } finally { refreshing = false; }
+  }
+
+  function connect(socket) {
+    const subscribe = (payload = {}, reply) => {
+      const respond = typeof reply === 'function' ? reply : () => {};
+      if (!isVerified(socket)) { respond({ error: 'Verification required' }); return; }
+      const roverId = typeof payload?.roverId === 'string' ? payload.roverId : null;
+      viewers.set(socket, { roverId, live: payload.live === true });
+      respond({ state: stateFor(socket, roverId) });
+      refresh().catch((error) => activity('Card update failed', { error: error.message }));
+    };
+    const action = async (payload = {}, reply) => {
+      const respond = typeof reply === 'function' ? reply : () => {};
+      try {
+        if (!isVerified(socket)) throw new Error('Verification required');
+        const roverId = String(payload.roverId || '');
+        if (payload.action === 'snapshot') {
+          if (!isAdmin(socket)) throw new Error('Administrator access required');
+          const model = await saveSnapshot(root);
+          models = await listModels(root);
+          lastCatalog = Date.now();
+          activity('Learner snapshot saved', { model: model.name });
+        } else if (payload.action === 'start') await driving.start(socket, roverId, String(payload.modelId || ''));
+        else if (payload.action === 'threshold') driving.setThreshold(socket, roverId, payload.value);
+        else if (payload.action === 'stop') {
+          const session = driving.getState(roverId);
+          if (!session || !(driving.canControl(socket, roverId) || session.socketId === socket.id)) throw new Error('You cannot stop this controller');
+          await driving.stop(roverId);
+        } else throw new Error('Unknown action');
+        viewers.set(socket, { roverId, live: viewers.get(socket)?.live || false });
+        respond({ state: stateFor(socket, roverId) });
+      } catch (error) { respond({ error: error.message }); }
+      refresh().catch((error) => activity('Card update failed', { error: error.message }));
+    };
+    const unsubscribe = () => viewers.delete(socket);
+    const disconnect = () => { unsubscribe(); detach(socket); };
+    const handlers = { 'roverLearning:subscribe': subscribe, 'roverLearning:action': action,
+      'roverLearning:unsubscribe': unsubscribe, disconnect };
+    for (const [name, handler] of Object.entries(handlers)) socket.on(name, handler);
+    bindings.set(socket, handlers);
+  }
+
+  function detach(socket) {
+    for (const [name, handler] of Object.entries(bindings.get(socket) || {})) socket.off(name, handler);
+    bindings.delete(socket);
+  }
+
+  io.on('connection', connect);
+  for (const socket of io.sockets.sockets.values()) connect(socket);
+  const timer = setInterval(() => { refresh().catch((error) => activity('Card update failed', { error: error.message })); }, 100);
+  refresh().catch((error) => activity('Catalog unavailable', { error: error.message }));
+  return {
+    close() {
+      closed = true;
+      clearInterval(timer);
+      io.off('connection', connect);
+      for (const socket of viewers.keys()) socket.emit('roverLearning:state', { available: false });
+      viewers.clear();
+      for (const socket of bindings.keys()) detach(socket);
+    },
+  };
+}
+
+module.exports = { createGateway };

server/src/services/roverLearningService/storage.js

@@ -0,0 +1,71 @@
+// Recording retention only. Published models and training state are never visited.
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { saveMetadata } = require('./recording');
+
+async function listRecordings(root) {
+  const recordings = [];
+  for (const entry of await fs.readdir(root, { withFileTypes: true })) {
+    // Directory names are generated, never rover IDs or user-supplied paths.
+    if (!entry.isDirectory() || !/^\d+-[0-9a-f-]{36}$/.test(entry.name)) continue;
+    const directory = path.join(root, entry.name);
+    let bytes = 0;
+    let files;
+    try { files = await fs.readdir(directory, { withFileTypes: true }); }
+    catch (error) { if (error.code === 'ENOENT') continue; throw error; }
+    for (const file of files) {
+      if (!file.isFile()) continue;
+      try { bytes += (await fs.stat(path.join(directory, file.name))).size; }
+      catch (error) {
+        // Dock-buffer pruning and atomic metadata replacement can remove a
+        // listed file before stat. Absence is normal; other I/O failures are not.
+        if (error.code !== 'ENOENT') throw error;
+      }
+    }
+    recordings.push({ id: entry.name, directory, bytes, startedAt: Number(entry.name.split('-')[0]) });
+  }
+  return recordings.sort((a, b) => a.startedAt - b.startedAt);
+}
+
+async function recoverRecordings(root) {
+  for (const recording of await listRecordings(root)) {
+    let metadata;
+    try {
+      metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8'));
+    } catch (error) {
+      if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
+      // An interrupted initial metadata write is retained as an unusable session
+      // until ordinary retention removes it, rather than blocking all startup.
+      metadata = { version: 1, id: recording.id, startedAt: recording.startedAt };
+    }
+    if (metadata.bufferOnly) {
+      await fs.rm(recording.directory, { recursive: true, force: true });
+      continue;
+    }
+    if (metadata.endedAt != null) continue;
+    await saveMetadata(recording.directory, {
+      ...metadata, endedAt: Date.now(), reason: 'interrupted',
+      error: 'Previous process did not close this session; final timestamps and video may be incomplete.',
+    });
+  }
+}
+
+async function enforceRetention(root, activeIds, maxBytes, minimumFreeBytes) {
+  const recordings = await listRecordings(root);
+  let bytes = recordings.reduce((total, recording) => total + recording.bytes, 0);
+  const stat = await fs.statfs(root);
+  let free = stat.bavail * stat.bsize;
+  for (const recording of recordings) {
+    if (bytes < maxBytes && free >= minimumFreeBytes) break;
+    if (activeIds.has(recording.id)) continue;
+    await fs.rm(recording.directory, { recursive: true });
+    bytes -= recording.bytes;
+    // Re-read actual free space; sparse/compressed files and other services can
+    // make a deleted file's logical size a misleading estimate of freed space.
+    const current = await fs.statfs(root);
+    free = current.bavail * current.bsize;
+  }
+  return { bytes, free, paused: bytes >= maxBytes || free < minimumFreeBytes };
+}
+
+module.exports = { recoverRecordings, enforceRetention, listRecordings };

server/src/services/roverLearningService/training.js

@@ -0,0 +1,213 @@
+// One owned finite worker at a time. Leases are acquired in the runtime's
+// serialized maintenance pass, before retention can next inspect recordings.
+const { errorDetails } = require('./errors');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+const { randomInt, createHash } = require('node:crypto');
+const { adoptCheckpoint } = require('./models');
+const { listRecordings } = require('./storage');
+
+function createTrainer({ root, config, actionThreshold, logger }) {
+  const leases = new Set();
+  const trainingRoot = path.join(root, 'training');
+  let ledger = {};
+  const history = [];
+  let child = null;
+  let completion = Promise.resolve();
+  let stopped = false;
+  let nextRun = 0;
+  let startupError = false;
+  let state = { status: config.enabled ? 'waiting' : 'disabled', steps: 0, examples: 0 };
+
+  async function start() {
+    if (!config.enabled) return;
+    try {
+      await fs.mkdir(trainingRoot, { recursive: true });
+      await fs.mkdir(path.join(root, 'models'), { recursive: true });
+      // Only unpublished staging directories from an interrupted old worker
+      // are disposable. Permanent UUID model directories are never removed.
+      for (const entry of await fs.readdir(path.join(root, 'models'), { withFileTypes: true })) {
+        if (entry.isDirectory() && /^\.[0-9a-f-]{36}\.pending$/.test(entry.name)) {
+          await fs.rm(path.join(root, 'models', entry.name), { recursive: true });
+        }
+      }
+      try {
+        const saved = JSON.parse(await fs.readFile(path.join(trainingRoot, 'schedule-v6.json'), 'utf8'));
+        ledger = saved.sessions;
+        state = { ...saved.progress, status: 'waiting' };
+        if (!ledger || typeof ledger !== 'object') throw new Error('Invalid training schedule');
+        // Retry this reader-specific rejection once after the fixed-grid repair.
+        // Keep optimizer progress and unrelated recording failures intact.
+        if (saved.samplingVersion !== 2) {
+          for (const [id, entry] of Object.entries(ledger)) {
+            if (entry.rejected === 'No complete, aligned observation/action windows') delete ledger[id];
+          }
+        }
+      } catch (error) { if (error.code !== 'ENOENT') throw error; }
+    } catch (error) {
+      startupError = true;
+      state = { ...state, status: 'error', error: error.message };
+      logger.error('Training initialization failed; recording remains available', { error: error.message, failure: errorDetails(error, 'training initialization') });
+    }
+  }
+
+  async function saveLedger() {
+    const filename = path.join(trainingRoot, 'schedule-v6.json');
+    await fs.writeFile(`${filename}.tmp`, JSON.stringify({ samplingVersion: 2, sessions: ledger, progress: state }));
+    await fs.rename(`${filename}.tmp`, filename);
+  }
+
+  async function tick(activeIds) {
+    if (!config.enabled || startupError || stopped || child || Date.now() < nextRun) return;
+    nextRun = Date.now() + config.intervalSeconds * 1000;
+    const recordings = await listRecordings(path.join(root, 'recordings'));
+    const present = new Set(recordings.map((recording) => recording.id));
+    // Scheduling history is bounded by recording retention, unlike models.
+    for (const id of Object.keys(ledger)) if (!present.has(id)) delete ledger[id];
+    const eligible = [];
+    for (const recording of recordings) {
+      if (activeIds.has(recording.id) || ledger[recording.id]?.rejected) continue;
+      try {
+        const metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8'));
+        if (metadata.bufferOnly || !metadata.endedAt || metadata.error || metadata.droppedEvents || metadata.reason === 'interrupted') continue;
+        eligible.push(recording.id);
+      } catch (error) {
+        ledger[recording.id] = { rejected: `Unreadable session metadata: ${error.message}` };
+      }
+    }
+    // Random cohorts mix old and new retained sessions without repeatedly
+    // favoring the first files in directory order.
+    for (let index = eligible.length - 1; index > 0; index -= 1) {
+      const other = randomInt(index + 1);
+      [eligible[index], eligible[other]] = [eligible[other], eligible[index]];
+    }
+    // Match workers/train.py exactly. Never move a previously trained session
+    // into evaluation just to fill a slot: that would contaminate the holdout.
+    const partitions = { training: [], heldOut: [] };
+    for (const id of eligible) {
+      const heldOut = parseInt(createHash('sha256').update(id).digest('hex').slice(0, 8), 16) % 5 === 0;
+      partitions[heldOut ? 'heldOut' : 'training'].push(id);
+    }
+    const trainingSessions = partitions.training.slice(0, partitions.heldOut.length ? 3 : 4);
+    const heldOutSessions = trainingSessions.length ? partitions.heldOut.slice(0, 2) : [];
+    const selected = [...trainingSessions, ...heldOutSessions];
+    const selection = {
+      atUnixMs: Date.now(),
+      availableTraining: partitions.training.length, availableHeldOut: partitions.heldOut.length,
+      selectedTraining: trainingSessions.length, selectedHeldOut: heldOutSessions.length,
+      reason: !trainingSessions.length ? 'Waiting for completed usable training-partition recordings'
+        : !heldOutSessions.length ? 'Training without evaluation: no eligible held-out recordings' : null,
+    };
+    state = { ...state, selection };
+    nextRun = Date.now() + config.intervalSeconds * 1000;
+    if (!selected.length) {
+      state = { ...state, status: 'waiting', reason: selection.reason };
+      await saveLedger();
+      return;
+    }
+    const jobPath = path.join(trainingRoot, 'job.json');
+    await fs.writeFile(jobPath, JSON.stringify({ root, config, actionThreshold, sessions: selected }));
+    if (stopped) return;
+    selected.forEach((id) => leases.add(id));
+    state = { ...state, status: 'training', reason: selection.reason, sessions: selected };
+    state.startedAt = Date.now();
+    child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1', OMP_NUM_THREADS: String(config.threads), MKL_NUM_THREADS: String(config.threads) },
+    });
+    let output = '';
+    let stderr = '';
+    let result = null;
+    let errorMessage = null;
+    let forcedTimer;
+    const worker = child;
+    const deadline = setTimeout(() => {
+      worker.kill('SIGTERM');
+      forcedTimer = setTimeout(() => worker.kill('SIGKILL'), 5000);
+    }, (config.maxJobSeconds + 15) * 1000);
+    child.on('error', (error) => { errorMessage = error.message; });
+    child.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-8192); });
+    child.stdout.on('data', (chunk) => {
+      output += chunk;
+      if (output.length > 1024 * 1024) {
+        errorMessage = 'Training worker output exceeded protocol bound';
+        worker.kill('SIGKILL');
+        return;
+      }
+      let newline;
+      while ((newline = output.indexOf('\n')) >= 0) {
+        const line = output.slice(0, newline);
+        output = output.slice(newline + 1);
+        try {
+          const message = JSON.parse(line);
+          if (message.kind === 'complete') result = message;
+          if (message.kind === 'error') errorMessage = message.message;
+          if (message.kind === 'skipped') logger.warn('Training skipped recording', message);
+          if (message.kind === 'dataset') state = { ...state, lastSampledWindows: message.sampled, lastAvailableWindows: message.windows, dataset: message.diagnostics };
+          if (message.kind === 'progress') state = { ...state, steps: message.steps, examples: message.examples, loss: message.loss, losses: message.losses };
+        } catch { errorMessage = 'Invalid training worker message'; }
+      }
+    });
+    completion = new Promise((resolve) => {
+      child.once('close', async (code, signal) => {
+        clearTimeout(deadline);
+        clearTimeout(forcedTimer);
+        try {
+          if (code !== 0 || errorMessage || !result) {
+            state = { ...state, status: stopped ? 'stopped' : 'error', error: errorMessage || `Worker exited ${code ?? signal}` };
+            if (!stopped) logger.error('Training job failed', { error: state.error, stderr });
+          } else {
+            for (const [id, reason] of Object.entries(result.rejected || {})) ledger[id] = { rejected: reason };
+            if (result.published) await adoptCheckpoint(root, result.published, result.promote);
+            if (result.trained) {
+              for (const id of result.accepted) ledger[id] = { rounds: (ledger[id]?.rounds || 0) + 1 };
+            }
+            state = { ...state, status: stopped ? 'stopped' : 'waiting', steps: result.steps ?? state.steps,
+              completedAt: Date.now(), elapsedSeconds: result.elapsedSeconds ?? null, error: null,
+              examples: result.examples ?? state.examples, loss: result.loss ?? null,
+              evaluatedSteps: result.trained ? result.steps : state.evaluatedSteps,
+              diagnostics: result.diagnostics ?? state.diagnostics, publication: result.publication ?? state.publication, evaluation: result.evaluation ?? state.evaluation,
+              reason: result.reason ?? 'Waiting for the next scheduled training job' };
+            await saveLedger();
+            logger.info('Training job completed', { ...state, model: result.published?.name || null });
+          }
+        } catch (error) {
+          state = { ...state, status: 'error', error: error.message };
+          logger.error('Training result persistence failed', { error: error.message, failure: errorDetails(error, 'training result persistence') });
+        } finally {
+          history.push({ at: Date.now(), status: state.status, steps: state.steps, loss: state.loss,
+            losses: state.losses || null, evaluation: state.evaluation ? Object.fromEntries(
+              ['status', 'windows', 'precision', 'recall', 'f1', 'predictedActions', 'positiveActions', 'numericMse', 'missedStops', 'gateReasons']
+                .map((key) => [key, state.evaluation[key]])) : null,
+            elapsedSeconds: state.elapsedSeconds || null });
+          if (history.length > 20) history.shift();
+          leases.clear();
+          child = null;
+          nextRun = Date.now() + config.intervalSeconds * 1000;
+          resolve();
+        }
+      });
+    });
+  }
+
+  async function cancel() {
+    if (!child) return;
+    const worker = child;
+    worker.kill('SIGTERM');
+    const timer = setTimeout(() => worker.kill('SIGKILL'), 5000);
+    try { await completion; } finally { clearTimeout(timer); }
+  }
+
+  return {
+    leases, start, tick, cancel,
+    getState: () => ({ ...state, history, nextAttemptAt: nextRun || null, leasedSessions: leases.size,
+      visitedSessions: Object.values(ledger).filter((entry) => entry.rounds).length,
+      skippedSessions: Object.values(ledger).filter((entry) => entry.rejected).length,
+      rejectionReasons: [...new Set(Object.values(ledger).filter((entry) => entry.rejected).map((entry) => entry.rejected))].slice(0, 10) }),
+    stop: async () => { stopped = true; await cancel(); },
+    kill: () => child?.kill('SIGKILL'),
+  };
+}
+
+module.exports = { createTrainer };

server/src/services/roverLearningService/workers/dataset.py

@@ -0,0 +1,223 @@
+"""Read closed recordings with causal state/action alignment and bounded memory."""
+import bisect
+import collections
+import copy
+import json
+import math
+from pathlib import Path
+
+import av
+import numpy as np
+from observations import decision_frames, history_frames
+
+from policy import SLOTS, VALUES, HEIGHT, WIDTH, SENSORS, command_family, is_stop, encode_command, sensor_vector
+
+
+def load_events(directory):
+    metadata = json.loads((directory / 'session.json').read_text())
+    if (metadata.get('version') != 1 or not metadata.get('endedAt') or metadata.get('error')
+            or metadata.get('droppedEvents') or metadata.get('reason') == 'interrupted'):
+        raise ValueError('Incomplete/interrupted recording or lost events')
+    if metadata.get('video', {}).get('origin') != 'unix':
+        raise ValueError('Unsupported video time origin')
+    video_result = metadata.get('video', {}).get('result') or {}
+    if video_result.get('signal') == 'SIGKILL' or video_result.get('code') not in (0, 255, None):
+        raise ValueError('Video recorder failed or was forcibly killed')
+    event_file = directory / 'events.ndjson'
+    if event_file.stat().st_size > 64 * 1024 * 1024:
+        raise ValueError('Event file exceeds 64 MiB read bound')
+    events = []
+    with event_file.open() as stream:
+        for line in stream:
+            event = json.loads(line)
+            if event['kind'] == 'captureGap':
+                raise ValueError('Recording contains a capture gap')
+            if not isinstance(event.get('ts'), (int, float)) or not math.isfinite(event['ts']):
+                raise ValueError('Invalid event timestamp')
+            events.append(event)
+    if not events or not any(e['kind'] == 'sessionEnd' for e in events):
+        raise ValueError('Missing completed event stream')
+    # File order can differ across rotation buffering. Monotonic time detects
+    # wall-clock steps; sorting Unix timestamps alone would hide those steps.
+    timed = sorted(events, key=lambda e: int(e['monotonicNs']))
+    origin = timed[0]
+    for event in timed:
+        drift = (event['ts'] - origin['ts']) - (int(event['monotonicNs']) - int(origin['monotonicNs'])) / 1e6
+        if abs(drift) > 250:
+            raise ValueError('Server wall clock shifted during recording')
+    return metadata, sorted(events, key=lambda e: e['ts'])
+
+
+def prepare(directory, schema, stats, extend):
+    metadata, events = load_events(directory)
+    requests = {}
+    accepted = set()
+    failed = set()
+    sensors, dispatches, boundaries = [], [], []
+    for event in events:
+        data = event['data']
+        if event['kind'] == 'sensor':
+            sensors.append((event['ts'], sensor_vector(data.get('sensors') or {})))
+        if event['kind'] in ('driver', 'turn:activeDriver', 'lock', 'private'):
+            boundaries.append(event['ts'])
+        if event['kind'] != 'command':
+            continue
+        if data.get('phase') == 'requested' and data.get('source') == 'client':
+            requests[data.get('requestId')] = data
+        if data.get('phase') == 'result' and data.get('result', {}).get('id'):
+            accepted.add(data['result']['id'])
+        if data.get('phase') == 'acknowledged' and data.get('acknowledgement', {}).get('error'):
+            failed.add(data.get('commandId'))
+        if data.get('phase') == 'dispatched':
+            dispatches.append(data)
+    if not sensors:
+        raise ValueError('No sensor observations')
+    candidate = copy.deepcopy(schema)
+    history, targets, unsupported_times = [], [], []
+    unsupported = {}
+    dispatch_delays = []
+    stats.update({'sensorFrames': len(sensors), 'dispatches': len(dispatches),
+                  'failedCommands': len(failed), 'sensorPresentFraction': np.mean([row[1][len(SENSORS):] for row in sensors], axis=0).tolist()})
+    for data in dispatches:
+        command = data.get('command') or {}
+        if command.get('type') in ('reset', 'reboot', 'update'):
+            boundaries.append(data['ts'])
+        try:
+            encoded = encode_command(command, candidate, extend=extend)
+        except ValueError as error:
+            # Unrepresentable controls invalidate only nearby observations/targets,
+            # not the rest of a recording. Never turn them into negative labels.
+            request = requests.get(data.get('requestId'))
+            unsupported_times.extend([data['ts'], request['ts'] if request else data['ts']])
+            key = json.dumps(command, sort_keys=True)
+            if key not in unsupported and len(unsupported) < 32:
+                unsupported[key] = {'command': command, 'reason': str(error), 'count': 0}
+            if key in unsupported:
+                unsupported[key]['count'] += 1
+            continue
+        if encoded is None:
+            continue
+        stamp = data['ts']
+        history.append((stamp, *encoded))
+        request = requests.get(data.get('requestId'))
+        if (data.get('source') == 'client' and request
+                and data.get('commandId') in accepted and data.get('commandId') not in failed):
+            target_time = request['ts']
+            if not 0 <= stamp - target_time <= 1000:
+                raise ValueError('Invalid request/dispatch timing')
+            dispatch_delays.append(stamp - target_time)
+            targets.append((target_time, *encoded))
+        else:
+            # Server safety overrides belong in history, never demonstration
+            # targets. Exclude windows around them rather than teaching inaction.
+            boundaries.append(stamp)
+    stats['requestToDispatchMs'] = {'mean': sum(dispatch_delays) / max(1, len(dispatch_delays)),
+                                    'max': max(dispatch_delays, default=0)}
+    stats['acceptedCommandEvents'] = len(targets)
+    stats['representedCommandEvents'] = 0
+    stats['commandEventCounts'] = [0] * SLOTS
+    stats['unsupportedCommands'] = list(unsupported.values())
+    stats['unsupportedCommandEvents'] = len(unsupported_times) // 2
+    if not targets and not unsupported_times:
+        raise ValueError('No accepted client operating commands')
+    schema[:] = candidate
+    return metadata, sensors, sorted(history, key=lambda x: x[0]), sorted(targets, key=lambda x: x[0]), sorted(boundaries), sorted(unsupported_times)
+
+
+def windows(directory, schema, cancelled=lambda: False, stats=None, extend=True):
+    directory = Path(directory)
+    stats = stats if stats is not None else {}
+    metadata, sensors, history, targets, boundaries, unsupported_times = prepare(directory, schema, stats, extend)
+    stats.update({'usableWindows': 0, 'commandWindows': 0, 'stopWindows': 0, 'noCommandWindows': 0,
+                  'staleSensorWindows': 0, 'boundaryWindows': 0, 'maxSensorAgeMs': 0, 'maxFrameAgeMs': 0,
+                  'unsupportedWindows': 0, 'ambiguousWindows': 0, 'videoFrames': 0, 'maxVideoGapMs': 0})
+    sensor_times = [row[0] for row in sensors]
+    target_times = [row[0] for row in targets]
+    available = np.zeros(SLOTS, dtype=np.float32)
+    available[:len(schema)] = 1
+    command_values = np.zeros((SLOTS, VALUES), dtype=np.float32)
+    command_times = np.full(SLOTS, -np.inf)
+    cursor = 0
+    recent = collections.deque(maxlen=32)
+    last_pts = -math.inf
+    def frames():
+        nonlocal last_pts
+        files = metadata['video'].get('files', ['video.mkv'])
+        for name in files:
+            if Path(name).name != name:
+                raise ValueError('Invalid video segment path')
+            with av.open(str(directory / name)) as container:
+                stream = container.streams.video[0]
+                stream.codec_context.thread_count = 1
+                for frame in container.decode(stream):
+                    if frame.pts is None:
+                        raise ValueError('Video frame has no timestamp')
+                    stamp = float(frame.pts * frame.time_base) * 1000
+                    stats['videoFrames'] += 1
+                    if math.isfinite(last_pts):
+                        stats['maxVideoGapMs'] = max(stats['maxVideoGapMs'], stamp - last_pts)
+                    last_pts = stamp
+                    if not metadata['startedAt'] - 2000 <= stamp <= metadata['endedAt'] + 2000:
+                        raise ValueError('Video PTS is not aligned to session Unix time')
+                    yield frame
+
+    represented = set()
+    for stamp, frame, captured_at in decision_frames(frames()):
+        if cancelled():
+            return
+        if stamp < metadata.get('trainingStartedAt', metadata['startedAt']):
+            continue
+        while cursor < len(history) and history[cursor][0] < stamp:
+            when, slot, values = history[cursor]
+            # Do not let drive predictions simply copy earlier human/model drive commands.
+            if schema[slot]['template']['type'] != 'drive':
+                command_values[slot] = values
+                command_times[slot] = when
+            cursor += 1
+        sensor_index = bisect.bisect_left(sensor_times, stamp) - 1
+        if sensor_index < 0 or stamp - sensor_times[sensor_index] > 1000:
+            stats['staleSensorWindows'] += 1
+            recent.clear()
+            continue
+        stats['maxSensorAgeMs'] = max(stats['maxSensorAgeMs'], stamp - sensor_times[sensor_index])
+        stats['maxFrameAgeMs'] = max(stats['maxFrameAgeMs'], stamp - captured_at)
+        seen = np.isfinite(command_times).astype(np.float32)
+        ages = np.minimum((stamp - command_times) / 10000, 1).astype(np.float32)
+        state = np.concatenate((sensors[sensor_index][1], command_values.flatten(), ages, seen))
+        pixels = frame.to_ndarray(width=WIDTH, height=HEIGHT, format='rgb24').transpose(2, 0, 1).copy()
+        recent.append((stamp, pixels, state))
+        chosen = history_frames(recent, stamp)
+        if len(chosen) != 4 or stamp + 100 > metadata['endedAt']:
+            continue
+        if bisect.bisect_right(unsupported_times, stamp + 100) > bisect.bisect_left(unsupported_times, chosen[0][0]):
+            stats['unsupportedWindows'] += 1
+            continue
+        if bisect.bisect_right(boundaries, stamp + 100) > bisect.bisect_left(boundaries, chosen[0][0]):
+            stats['boundaryWindows'] += 1
+            continue
+        start = bisect.bisect_left(target_times, stamp)
+        end = bisect.bisect_left(target_times, stamp + 100)
+        actions = np.zeros(SLOTS, dtype=np.float32)
+        values = np.zeros((SLOTS, VALUES), dtype=np.float32)
+        ambiguous = False
+        families = set()
+        for _, slot, vector in targets[start:end]:
+            template = schema[slot]['template']
+            family = command_family(template)
+            if ((actions[slot] and (template['type'] in ('headlight', 'raw') or not np.array_equal(values[slot], vector)))
+                    or (not actions[slot] and family in families)):
+                ambiguous = True
+                break
+            families.add(family)
+            actions[slot], values[slot] = 1, vector
+        if ambiguous:
+            stats['ambiguousWindows'] += 1
+            continue
+        represented.update(range(start, end))
+        stats['representedCommandEvents'] = len(represented)
+        for slot in np.flatnonzero(actions):
+            stats['commandEventCounts'][slot] += 1
+        stats['usableWindows'] += 1
+        stats['commandWindows' if actions.any() else 'noCommandWindows'] += 1
+        stats['stopWindows'] += int(any(actions[slot] and is_stop(descriptor) for slot, descriptor in enumerate(schema)))
+        yield np.stack([row[1] for row in chosen]), np.stack([row[2] for row in chosen]), actions, values, available

server/src/services/roverLearningService/workers/infer.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Live policy worker. JSON state on stdin, proposals on stdout; no rover writes."""
+import argparse
+import collections
+import hashlib
+import json
+import math
+from pathlib import Path
+import sys
+import threading
+import time
+
+import av
+import numpy as np
+import torch
+from observations import decision_frames, history_frames
+
+from policy import Policy, VERSION, SLOTS, VALUES, HEIGHT, WIDTH, SENSORS, select_slots, decode_command, encode_command, sensor_vector
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--model', required=True)
+    parser.add_argument('--url', required=True)
+    parser.add_argument('--threads', type=int, default=4)
+    parser.add_argument('--threshold', type=float, default=0.7)
+    args = parser.parse_args()
+    torch.set_num_threads(args.threads)
+    torch.set_num_interop_threads(1)
+    directory = Path(args.model)
+    metadata = json.loads((directory / 'model.json').read_text())
+    if metadata['specification']['version'] != VERSION:
+        raise ValueError('Model specification is incompatible')
+    if hashlib.sha256((directory / 'weights.pt').read_bytes()).hexdigest() != metadata['sha256']:
+        raise ValueError('Model weights checksum mismatch')
+    schema = metadata['specification']['commands']
+    model = Policy().eval()
+    model.load_state_dict(torch.load(directory / 'weights.pt', map_location='cpu', weights_only=True))
+    observations = collections.deque(maxlen=2048)
+    mutex = threading.Lock()
+    closed = threading.Event()
+
+    def read_input():
+        try:
+            for line in sys.stdin:
+                event = json.loads(line)
+                with mutex:
+                    observations.append(event)
+        finally:
+            closed.set()
+
+    threading.Thread(target=read_input, daemon=True).start()
+    values = np.zeros((SLOTS, VALUES), dtype=np.float32)
+    last_commands = np.full(SLOTS, -np.inf)
+    sensors, sensor_time = {}, 0
+    history = collections.deque(maxlen=32)
+    latched = set()
+    with av.open(args.url, options={'rtsp_transport': 'tcp', 'timeout': '3000000',
+                                   'use_wallclock_as_timestamps': '1'}) as video:
+        stream = video.streams.video[0]
+        stream.codec_context.thread_count = 1
+        for stamp, frame, captured_at in decision_frames(video.decode(stream)):
+            if closed.is_set():
+                return
+            now = time.time() * 1000
+            if abs(now - captured_at) > 2000:
+                raise ValueError('Video time is stale or not aligned to server time')
+            with mutex:
+                while observations and observations[0]['ts'] < stamp:
+                    event = observations.popleft()
+                    if event['kind'] == 'settings':
+                        args.threshold = max(.05, min(.99, float(event['threshold'])))
+                    elif event['kind'] == 'sensor':
+                        sensors, sensor_time = event['data'], event['ts']
+                    elif event['kind'] == 'command' and event['data'].get('type') != 'drive':
+                        try:
+                            encoded = encode_command(event['data'], schema)
+                            if encoded:
+                                slot, vector = encoded
+                                values[slot], last_commands[slot] = vector, event['ts']
+                        except ValueError:
+                            pass  # Unseen manual controls do not stop autonomy.
+            if stamp - sensor_time > 1000:
+                history.clear()
+                continue
+            state = np.concatenate((sensor_vector(sensors), values.flatten(),
+                                    np.minimum((stamp - last_commands) / 10000, 1).astype(np.float32),
+                                    np.isfinite(last_commands).astype(np.float32)))
+            pixels = frame.to_ndarray(width=WIDTH, height=HEIGHT, format='rgb24').transpose(2, 0, 1).copy()
+            history.append((stamp, pixels, state))
+            selected = history_frames(history, stamp)
+            if len(selected) != 4:
+                continue
+            before = time.perf_counter()
+            with torch.inference_mode():
+                logits, numeric = model(torch.from_numpy(np.stack([row[1] for row in selected])).unsqueeze(0).float() / 255,
+                                        torch.from_numpy(np.stack([row[2] for row in selected])).unsqueeze(0))
+                probabilities = logits[0].sigmoid().tolist()
+                predictions = numeric[0].tolist()
+            if not all(math.isfinite(value) for value in probabilities):
+                raise ValueError('Nonfinite model output')
+            winners = select_slots(probabilities, schema, args.threshold)
+            active = {slot for slot in range(len(schema)) if probabilities[slot] >= args.threshold}
+            latched.intersection_update(active)
+            proposals, commands = [], []
+            for slot, descriptor in enumerate(schema):
+                command = decode_command(descriptor, predictions[slot])
+                discrete = command['type'] in ('headlight', 'raw')
+                reason = ('Below threshold' if slot not in active else
+                          'Competing action' if slot not in winners else
+                          'Held discrete action' if discrete and slot in latched else 'Proposed')
+                proposals.append({'slot': slot, 'score': probabilities[slot], 'command': command, 'reason': reason})
+                if reason == 'Proposed':
+                    commands.append(command)
+                if slot in winners and discrete:
+                    latched.add(slot)
+            print(json.dumps({'kind': 'prediction', 'frameAt': captured_at, 'decisionAt': stamp, 'sensorAt': sensor_time,
+                              'latencyMs': (time.perf_counter() - before) * 1000,
+                              'sensorPresent': sensor_vector(sensors)[len(SENSORS):].tolist(),
+                              'commands': commands, 'proposals': proposals, 'threshold': args.threshold}, allow_nan=False), flush=True)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as error:
+        print(json.dumps({'kind': 'error', 'message': str(error)}), flush=True)
+        raise SystemExit(1)

server/src/services/roverLearningService/workers/learning.py

@@ -0,0 +1,181 @@
+"""Training objective and bounded, observational learning diagnostics."""
+import copy
+import time
+
+import numpy as np
+import torch
+from torch.nn import functional as F
+
+from policy import SLOTS, VALUES, select_slots, decode_command, is_stop
+
+
+def loss_setup(samples, schema):
+    numeric_mask = torch.zeros(SLOTS, VALUES)
+    for slot, descriptor in enumerate(schema):
+        numeric_mask[slot, :len(descriptor['fields'])] = 1
+    positives = np.sum([sample[2] for sample in samples], axis=0)
+    opportunities = np.sum([sample[4] for sample in samples], axis=0)
+    weight = torch.from_numpy(np.clip((opportunities - positives) / np.maximum(positives, 1), 1, 20)).float()
+    return numeric_mask, weight
+
+
+def tensors(samples):
+    images, states, targets, values, available = [torch.from_numpy(np.stack(items)) for items in zip(*samples)]
+    return images.float().div_(255), states, targets, values, available
+
+
+def losses(logits, predictions, targets, values, available, numeric_mask, positive_weight):
+    events = F.binary_cross_entropy_with_logits(logits, targets, pos_weight=positive_weight, reduction='none')
+    events = (events * available).sum() / available.sum().clamp_min(1)
+    mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
+    parameters = ((predictions - values).square() * mask).sum() / mask.sum().clamp_min(1)
+    return events, parameters
+
+
+def distribution(values):
+    values = np.asarray(values, dtype=np.float64)
+    if not values.size:
+        return {'count': 0, 'mean': None, 'p50': None, 'p95': None, 'max': None}
+    return {'count': int(values.size), 'mean': float(values.mean()),
+            'p50': float(np.quantile(values, .5)), 'p95': float(np.quantile(values, .95)), 'max': float(values.max())}
+
+
+def decisions(scores, schema, threshold):
+    selected = np.zeros_like(scores, dtype=bool)
+    for row, vector in enumerate(scores):
+        for slot in select_slots(vector, schema, threshold):
+            selected[row, slot] = True
+    return selected
+
+
+def counts(predicted, actual):
+    matched = int((predicted & actual).sum())
+    proposed, wanted = int(predicted.sum()), int(actual.sum())
+    empty = ~actual.any(axis=1)
+    return {'predictedActions': proposed, 'positiveActions': wanted, 'matchedActions': matched,
+            'precision': matched / max(1, proposed), 'recall': matched / max(1, wanted),
+            'f1': 2 * matched / max(1, proposed + wanted),
+            'falsePositiveActions': proposed - matched,
+            'noCommandWindows': int(empty.sum()), 'unwantedCommandWindows': int((empty & predicted.any(axis=1)).sum())}
+
+
+def summarize(scores, actual, schema, threshold, frequencies):
+    active_scores = scores[:, :len(schema)]
+    labels = actual[:, :len(schema)]
+    thresholds = sorted(set([.1, .3, .5, .7, .9, threshold]))
+    sweep = [{'threshold': value, **counts(decisions(scores, schema, value), actual)} for value in thresholds]
+    frequent = np.broadcast_to(frequencies, scores.shape)
+    baseline = [{'threshold': value, **counts(decisions(frequent, schema, value), actual)} for value in thresholds]
+    return {'scoreOnHumanCommands': distribution(active_scores[labels]),
+            'scoreOnAbsentCommands': distribution(active_scores[~labels]),
+            'maxScoreOnNoCommandWindows': distribution(active_scores[~actual.any(axis=1)].max(axis=1)) if (~actual.any(axis=1)).any() else distribution([]),
+            'thresholds': sweep, 'baselines': {'sendNothing': counts(np.zeros_like(actual), actual),
+                'trainingFrequency': baseline, 'frequencySource': 'Training windows only; no held-out fitting'},
+            'perCommandScores': [{'slot': slot, 'onHumanCommand': distribution(scores[actual[:, slot], slot]),
+                                   'otherwise': distribution(scores[~actual[:, slot], slot])} for slot in range(len(schema))]}
+
+
+def evaluate(model, samples, schema, threshold, frequencies, batch_size, cancelled):
+    """Same score selection as runtime; isolated windows cannot measure latching."""
+    model.eval()
+    score_rows, target_rows = [], []
+    value_error = value_count = 0
+    decoded = {}
+    with torch.inference_mode():
+        for offset in range(0, len(samples), batch_size):
+            if cancelled():
+                break
+            images, states, targets, values, _ = tensors(samples[offset:offset + batch_size])
+            logits, numeric = model(images, states)
+            if not torch.isfinite(logits).all() or not torch.isfinite(numeric).all():
+                raise ValueError('Nonfinite evaluation output')
+            score_rows.append(logits.sigmoid().numpy())
+            target_rows.append(targets.numpy().astype(bool))
+            for row, slot in targets.nonzero().tolist():
+                descriptor = schema[slot]
+                n = len(descriptor['fields'])
+                value_error += (numeric[row, slot, :n] - values[row, slot, :n]).square().sum().item()
+                value_count += n
+                prediction = decode_command(descriptor, numeric[row, slot])
+                expected = decode_command(descriptor, values[row, slot])
+                for field in descriptor['fields']:
+                    a, b = prediction, expected
+                    for key in field['path']:
+                        a, b = a[key], b[key]
+                    key = descriptor['template']['type'] + '.' + '.'.join(map(str, field['path']))
+                    total, count = decoded.get(key, (0, 0))
+                    decoded[key] = (total + abs(a - b), count + 1)
+    if not score_rows:
+        return {'windows': 0, 'complete': False}
+    scores, actual = np.concatenate(score_rows), np.concatenate(target_rows)
+    predicted = decisions(scores, schema, threshold)
+    stop_slots = [i for i, d in enumerate(schema) if is_stop(d)]
+    move_slots = [i for i, d in enumerate(schema) if d['template']['type'] == 'drive' and not is_stop(d)]
+    stop_count = int(actual[:, stop_slots].sum())
+    stop_match = int((actual[:, stop_slots] & predicted[:, stop_slots]).sum())
+    move_count = int(actual[:, move_slots].sum())
+    move_match = int((actual[:, move_slots] & predicted[:, move_slots]).sum())
+    return {'windows': len(scores), 'complete': len(scores) == len(samples),
+            'stopEvents': stop_count, 'missedStops': stop_count - stop_match,
+            'stopRecall': stop_match / max(1, stop_count),
+            'falseStops': int((predicted[:, stop_slots] & ~actual[:, stop_slots]).sum()),
+            'movementEvents': move_count, 'movementRecall': move_match / max(1, move_count),
+            'commands': [{'slot': i, 'command': d['template'], 'actual': int(actual[:, i].sum()),
+                          'predicted': int(predicted[:, i].sum()),
+                          'matched': int((actual[:, i] & predicted[:, i]).sum())} for i, d in enumerate(schema)],
+            'numericMse': value_error / max(1, value_count),
+            'decodedParameterMaeBeforeServerLimits': {key: total / count for key, (total, count) in decoded.items()},
+            **counts(decisions(scores, schema, threshold), actual),
+            **summarize(scores, actual, schema, threshold, frequencies)}
+
+
+def probe(model, samples, schema, threshold, frequencies, cancelled, max_seconds=20):
+    """Try fitting a small real training subset on a disposable copy, never publish it."""
+    started = time.monotonic()
+    # Include both event and quiet windows; source is training only. No random
+    # state/optimizer changes escape into the continuing learner.
+    quiet = [row for row in samples if not row[2].any()][:4]
+    # A reservoir is grouped by category: taking its first events could probe
+    # only zero-speed drive and falsely suggest all controls were learnable.
+    by_slot = [[i for i, row in enumerate(samples) if row[2][slot]] for slot in range(len(schema))]
+    chosen = []
+    for rank in range(8):
+        for indices in by_slot:
+            if rank < len(indices) and indices[rank] not in chosen and len(chosen) < 8:
+                chosen.append(indices[rank])
+    subset = [samples[i] for i in chosen] + quiet
+    if not chosen:
+        return {'status': 'No command examples', 'steps': 0}
+    with torch.random.fork_rng(devices=[]):
+        candidate = copy.deepcopy(model)
+        optimizer = torch.optim.AdamW(candidate.parameters(), lr=.001)
+        numeric_mask, weights = loss_setup(subset, schema)
+        images, states, targets, values, available = tensors(subset)
+        initial_loss = final_loss = gradient_norm = None
+        completed = 0
+        expired = lambda: cancelled() or time.monotonic() - started >= max_seconds
+        for _ in range(80):
+            if expired():
+                break
+            candidate.train()
+            optimizer.zero_grad(set_to_none=True)
+            logits, predictions = candidate(images, states)
+            event_loss, value_loss = losses(logits, predictions, targets, values, available, numeric_mask, weights)
+            loss = event_loss + value_loss
+            if not torch.isfinite(loss):
+                raise ValueError('Nonfinite learning probe loss')
+            if initial_loss is None:
+                initial_loss = loss.item()
+            loss.backward()
+            norm = torch.nn.utils.clip_grad_norm_(candidate.parameters(), 1)
+            if not torch.isfinite(norm):
+                raise ValueError('Nonfinite learning probe gradient')
+            gradient_norm = float(norm)
+            optimizer.step()
+            final_loss = loss.item()
+            completed += 1
+        result = evaluate(candidate, subset, schema, threshold, frequencies, 16, cancelled)
+    return {'status': 'completed' if completed == 80 else 'budget limited', 'steps': completed,
+            'windows': len(subset), 'initialLoss': initial_loss, 'lastStepLoss': final_loss,
+            'lastGradientNormBeforeClipping': gradient_norm, 'elapsedSeconds': time.monotonic() - started,
+            'evaluation': result, 'meaning': 'Memorization on a disposable copy of training data, not generalization'}

server/src/services/roverLearningService/workers/observations.py

@@ -0,0 +1,45 @@
+"""Shared causal 10 Hz sampling for recorded and live video."""
+import math
+
+
+def decision_frames(frames):
+    """Yield decision time, latest past image, and its actual capture timestamp.
+
+    Sampling every third/fourth decoded frame leaves holes at rates such as 25
+    FPS. A fixed grid covers each 100ms command interval exactly once. A frame
+    after the decision can release the pending tick, but is never its input.
+    """
+    previous = None
+    previous_time = None
+    tick = None
+    for frame in frames:
+        if frame.pts is None:
+            raise ValueError('Video frame has no timestamp')
+        stamp = float(frame.pts * frame.time_base) * 1000
+        if not math.isfinite(stamp) or (previous_time is not None and stamp < previous_time):
+            raise ValueError('Invalid/nonmonotonic video timestamps')
+        if tick is None:
+            tick = stamp
+        while tick < stamp - .01:
+            # Do not synthesize observations over missing/stalled video. History
+            # selection will reject a temporal gap until fresh history exists.
+            if previous is not None and tick - previous_time <= 200:
+                yield tick, previous, previous_time
+            elif stamp - tick > 200:
+                tick += max(1, math.floor((stamp - tick) / 100)) * 100
+                continue
+            tick += 100
+        if abs(stamp - tick) <= .01:
+            yield tick, frame, stamp
+            tick += 100
+        previous, previous_time = frame, stamp
+
+
+def history_frames(history, stamp):
+    selected = []
+    for offset in (900, 600, 300, 0):
+        candidates = [row for row in history if row[0] <= stamp - offset + .01]
+        if not candidates or stamp - offset - candidates[-1][0] > 200:
+            return []
+        selected.append(candidates[-1])
+    return selected

server/src/services/roverLearningService/workers/policy.py

@@ -0,0 +1,192 @@
+"""Versioned, fixed-size vision/control policy and command encoding.
+
+Slots describe command shapes, not arbitrary executable instructions. Numeric
+leaves are regressed; string/bool/list values remain exact template constants.
+No command is sent by this module. Execution must use server authorization.
+"""
+import base64
+import itertools
+import copy
+import json
+import math
+
+import numpy as np
+import torch
+from torch import nn
+
+VERSION = 5
+SLOTS = 64
+VALUES = 8
+FRAMES = 4
+HEIGHT, WIDTH = 120, 160
+OPERATING_TYPES = {"drive", "servo", "headlight", "raw"}
+MODE_COMMANDS = tuple(base64.b64encode(bytes([opcode])).decode() for opcode in (128, 131, 132, 143))
+SENSORS = [
+    ("bumpsAndWheelDrops.bumpLeft", 1), ("bumpsAndWheelDrops.bumpRight", 1),
+    ("bumpsAndWheelDrops.wheelDropLeft", 1), ("bumpsAndWheelDrops.wheelDropRight", 1),
+    ("cliffLeft", 1), ("cliffFrontLeft", 1), ("cliffFrontRight", 1), ("cliffRight", 1),
+    ("wall", 1), ("virtualWall", 1), ("wallSignal", 4096),
+    ("voltageMv", 16000), ("currentMa", 3000), ("batteryChargeMah", 4000),
+    ("batteryCapacityMah", 4000), ("batteryTemperatureC", 50),
+    ("chargingState.code", 5), ("chargingSources.homeBase", 1), ("oiMode.code", 4),
+    ("wheelLeftCurrentMa", 2000), ("wheelRightCurrentMa", 2000),
+    ("mainBrushCurrentMa", 2000), ("sideBrushCurrentMa", 2000),
+    ("lightBumpLeftSignal", 4096), ("lightBumpRightSignal", 4096),
+    ("cliffLeftSignal", 4096), ("cliffRightSignal", 4096),
+    ("distanceMm", 500), ("angleDeg", 180), ("stasis", 1),
+]
+STATE_SIZE = len(SENSORS) * 2 + SLOTS * (VALUES + 2)
+
+
+def normalize(value, scale):
+    return math.atan(float(value) / scale) * 2 / math.pi
+
+
+def sensor_vector(sensors):
+    values, present = [], []
+    for name, scale in SENSORS:
+        value = sensors
+        for key in name.split('.'):
+            value = value.get(key) if isinstance(value, dict) else None
+        known = isinstance(value, (int, float)) and math.isfinite(value)
+        values.append(normalize(value, scale) if known else 0)
+        present.append(float(known))
+    return np.array(values + present, dtype=np.float32)
+
+
+def describe_command(command):
+    if command.get('type') not in OPERATING_TYPES:
+        return None
+    # Raw is restricted to the existing mode/dock buttons, not arbitrary OI
+    # packets that could reintroduce excluded actuators or songs.
+    if command['type'] == 'raw' and command.get('raw') not in MODE_COMMANDS:
+        return None
+    if len(json.dumps(command)) > 4096:
+        raise ValueError('Operating command exceeds 4096-byte schema limit')
+    values, fields = [], []
+
+    def visit(value, path=()):
+        if isinstance(value, dict):
+            return {key: visit(item, (*path, key)) for key, item in sorted(value.items())}
+        # Exact actuator zeros preserve demonstrated releases.
+        if value == 0:
+            return copy.deepcopy(value)
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            if not math.isfinite(value) or len(values) >= VALUES:
+                raise ValueError('Nonfinite command or too many numeric control fields')
+            scale = {'drive': 500, 'servo': 180}.get(command['type'], 1)
+            fields.append({'path': list(path), 'scale': scale, 'integer': command['type'] in ('drive', 'servo') or isinstance(value, int)})
+            values.append(normalize(value, scale))
+            return {'$number': len(values) - 1}
+        # Mode bytes and named actions remain exact constants.
+        return copy.deepcopy(value)
+
+    template = visit(command)
+    return {'template': template, 'fields': fields}, values
+
+
+def encode_command(command, schema, extend=False):
+    described = describe_command(command)
+    if described is None:
+        return None
+    descriptor, values = described
+    try:
+        slot = schema.index(descriptor)
+    except ValueError:
+        if not extend:
+            raise ValueError('Command shape missing from model schema')
+        if len(schema) >= SLOTS:
+            raise ValueError('64 command shapes exhausted; affected windows excluded')
+        slot = len(schema)
+        schema.append(descriptor)
+    vector = np.zeros(VALUES, dtype=np.float32)
+    vector[:len(values)] = values
+    return slot, vector
+
+
+def initial_schema():
+    # Match the retained commandPipeline.js controls. Values describe fields,
+    # not artificial demonstrations; zero-speed drive remains a drive command.
+    commands = []
+    for left, right in itertools.product((0, 1), repeat=2):
+        commands.append({'type': 'drive', 'driveDirect': {'left': left, 'right': right}})
+    for angle in (0, 1):
+        commands.append({'type': 'servo', 'servo': {'angle': angle}})
+    for action in ('on', 'off', 'toggle'):
+        commands.append({'type': 'headlight', 'headlight': {'action': action}})
+    for raw in MODE_COMMANDS:
+        commands.append({'type': 'raw', 'raw': raw})
+    schema = []
+    for command in commands:
+        encode_command(command, schema, extend=True)
+    return schema
+
+
+def command_family(command):
+    return (command['type'],)
+
+
+def select_slots(scores, schema, threshold):
+    winners = {}
+    for slot, descriptor in enumerate(schema):
+        if scores[slot] < threshold:
+            continue
+        family = command_family(descriptor['template'])
+        if family not in winners or scores[slot] > scores[winners[family]]:
+            winners[family] = slot
+    return set(winners.values())
+
+
+def decode_command(descriptor, values):
+    command = copy.deepcopy(descriptor['template'])
+    for index, field in enumerate(descriptor['fields']):
+        number = float(values[index])
+        if not math.isfinite(number):
+            raise ValueError('Nonfinite control value')
+        number = math.tan(max(-.98, min(.98, number)) * math.pi / 2) * field['scale']
+        target = command
+        for key in field['path'][:-1]:
+            target = target[key]
+        target[field['path'][-1]] = round(number) if field['integer'] else number
+    return command
+
+
+def is_stop(descriptor):
+    command = descriptor['template']
+    direct = command.get('driveDirect') or {}
+    return command['type'] == 'drive' and direct.get('left') == 0 and direct.get('right') == 0
+
+
+class Policy(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.vision = nn.Sequential(
+            nn.Conv2d(3, 24, 5, 2, 2), nn.ReLU(),
+            nn.Conv2d(24, 48, 3, 2, 1), nn.ReLU(),
+            nn.Conv2d(48, 96, 3, 2, 1), nn.ReLU(),
+            nn.Conv2d(96, 128, 3, 2, 1), nn.ReLU(),
+            nn.AdaptiveAvgPool2d((3, 4)), nn.Flatten(),
+            nn.Linear(128 * 3 * 4, 128), nn.ReLU(),
+        )
+        self.history = nn.GRU(128 + STATE_SIZE, 128, batch_first=True)
+        self.actions = nn.Linear(128, SLOTS)
+        self.values = nn.Linear(128, SLOTS * VALUES)
+
+    def forward(self, images, state):
+        batch, frames = images.shape[:2]
+        encoded = self.vision(images.reshape(-1, 3, HEIGHT, WIDTH)).reshape(batch, frames, -1)
+        history, _ = self.history(torch.cat((encoded, state), dim=-1))
+        last = history[:, -1]
+        return self.actions(last), self.values(last).reshape(batch, SLOTS, VALUES).tanh()
+
+
+def specification(schema):
+    return {'version': VERSION, 'architecture': 'spatial-cnn-gru-events-v5',
+            'frames': FRAMES, 'height': HEIGHT, 'width': WIDTH,
+            'historyStrideMs': 300, 'decisionIntervalMs': 100,
+            'stateSize': STATE_SIZE, 'sensorFields': SENSORS,
+            'normalization': '2/pi * atan(value/scale); missing sensors have mask=0',
+            'maxCommandShapes': SLOTS, 'numericFieldsPerShape': VALUES,
+            'commands': schema, 'actionSemantics': 'All controls are command events in the next 100ms; no event preserves existing rover state',
+            'driveCommandHistoryInput': False, 'zeroNumericValues': 'Exact template constants',
+            'timing': 'Server-receipt alignment; browser display latency unmeasured'}

server/src/services/roverLearningService/workers/requirements.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.14.0+cpu
+numpy==2.5.3
+av==18.1.0

server/src/services/roverLearningService/workers/train.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+"""One finite CPU training job; Node owns scheduling, leases, and shutdown.
+
+Usage: python train.py --job /path/to/job.json
+Dependencies are packaged at image build time, never installed at runtime.
+"""
+import argparse
+import copy
+import hashlib
+import json
+import os
+from pathlib import Path
+import random
+import resource
+import signal
+import time
+import uuid
+
+
+def emit(kind, **data):
+    print(json.dumps({'kind': kind, **data}, allow_nan=False), flush=True)
+
+
+def atomic_torch_save(torch, value, path):
+    temporary = path.with_suffix('.tmp')
+    with temporary.open('wb') as stream:
+        torch.save(value, stream)
+        stream.flush()
+        os.fsync(stream.fileno())
+    temporary.replace(path)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--job')
+    parser.add_argument('--check-dependencies', action='store_true', help='Describe the installed production policy and exit without creating storage')
+    args = parser.parse_args()
+    # --help works even before Python dependencies are installed.
+    import numpy as np
+    import av
+    import torch
+    from learning import loss_setup, tensors, losses as policy_losses, evaluate, probe
+    from dataset import windows
+    from policy import Policy, VERSION, SLOTS, VALUES, specification, is_stop, select_slots, initial_schema
+
+    if args.check_dependencies:
+        emit('dependencies', torch=str(torch.__version__), numpy=str(np.__version__), av=str(av.__version__),
+             parameters=sum(p.numel() for p in Policy().parameters()), specification=specification(initial_schema()))
+        return
+    if not args.job:
+        parser.error('--job is required unless --check-dependencies is used')
+
+    job = json.loads(Path(args.job).read_text())
+    root = Path(job['root'])
+    config = job['config']
+    torch.set_num_threads(config['threads'])
+    torch.set_num_interop_threads(1)
+    # Training yields CPU priority to the Node/media processes. Thread count is
+    # an additional bound, not a claim to reserve specific physical cores.
+    os.nice(10)
+    stopped = False
+
+    def stop(_signal, _frame):
+        nonlocal stopped
+        stopped = True
+
+    signal.signal(signal.SIGTERM, stop)
+    signal.signal(signal.SIGINT, stop)
+    started = time.monotonic()
+    cancelled = lambda: stopped or time.monotonic() - started >= config['maxJobSeconds']
+    model = Policy().cpu()
+    optimizer = torch.optim.AdamW(model.parameters(), lr=0.0001)
+    # The command-event policy has different inputs, targets and weights. Keep its
+    # optimizer lineage separate; reuse recordings with their unchanged partition.
+    resume_path = root / 'training' / 'resume-v6.pt'
+    schema, steps, examples, last_published = initial_schema(), 0, 0, 0
+    trained_rovers = set()
+    control_profiles = []
+    distinct_windows = {}
+    published_windows = 0
+    last_probe_steps, learning_probe = 0, None
+    if resume_path.exists():
+        saved = torch.load(resume_path, map_location='cpu', weights_only=True)
+        if saved['version'] != VERSION:
+            raise ValueError('Resume schema incompatible; refusing to reinterpret existing weights')
+        model.load_state_dict(saved['weights'])
+        optimizer.load_state_dict(saved['optimizer'])
+        schema = saved['schema']
+        steps, examples = saved['steps'], saved['examples']
+        last_published = saved['lastPublished']
+        trained_rovers = set(saved['trainedRovers'])
+        control_profiles = saved.get('controlProfiles', [])
+        distinct_windows = saved.get('distinctWindows', {})
+        published_windows = saved.get('publishedWindows', 0)
+        last_probe_steps = saved.get('lastProbeSteps', 0)
+        learning_probe = saved.get('learningProbe')
+    cpu_started = time.process_time()
+    decode_started = time.monotonic()
+    dataset_stats = []
+    initial_steps = steps
+    pool, validation, accepted, rejected = [], [], [], {}
+    observed_windows = {}
+    sessions = list(job['sessions'])
+    random.shuffle(sessions)
+    is_held_out = lambda session_id: int(hashlib.sha256(session_id.encode()).hexdigest()[:8], 16) % 5 == 0
+    sessions.sort(key=is_held_out)
+    per_session = max(1, config['maxSamples'] // max(1, len(sessions)))
+    for session_id in sessions:
+        if cancelled():
+            break
+        candidate_schema = copy.deepcopy(schema)
+        samples, seen = [], 0
+        stats = {}
+        buckets = {key: [] for key in ('stop', 'command', 'none')}
+        counts = dict.fromkeys(buckets, 0)
+        held_out = is_held_out(session_id)
+        try:
+            directory = root / 'recordings' / session_id
+            for sample in windows(directory, candidate_schema, cancelled, stats, extend=not held_out):
+                seen += 1
+                # Keep evaluation naturally distributed. Training reserves space
+                # for recorded commands and releases so periods without commands
+                # cannot consume the entire sample budget. No scripted driving required.
+                if held_out:
+                    reservoir, count, capacity = samples, seen, per_session
+                else:
+                    category = ('stop' if any(sample[2][slot] and is_stop(descriptor)
+                                              for slot, descriptor in enumerate(candidate_schema)) else
+                                'command' if sample[2].any() else 'none')
+                    counts[category] += 1
+                    reservoir, count, capacity = buckets[category], counts[category], max(1, per_session // 3)
+                if len(reservoir) < capacity:
+                    reservoir.append(sample)
+                else:
+                    index = random.randrange(count)
+                    if index < capacity:
+                        reservoir[index] = sample
+            if not held_out:
+                samples = [sample for bucket in buckets.values() for sample in bucket]
+            if cancelled():
+                break  # Do not call a partially decoded recording validated.
+            if not samples:
+                if held_out and stats.get('unsupportedCommandEvents'):
+                    dataset_stats.append({**stats, 'partition': 'heldOut', 'sampledWindows': 0,
+                                          'skipReason': 'No supported aligned windows'})
+                    continue
+                raise ValueError('No complete, aligned observation/action windows')
+            # Entire sessions stay in one partition across all jobs. Neighboring
+            # video windows must never straddle training and validation.
+            held_out = is_held_out(session_id)
+            if held_out:
+                validation.extend(samples)
+            else:
+                schema = candidate_schema
+                pool.extend(samples)
+                observed_windows[session_id] = seen
+            accepted.append(session_id)
+            metadata = json.loads((directory / 'session.json').read_text())
+            if not held_out:
+                trained_rovers.add(str(metadata['roverId']))
+            profile = metadata.get('controlProfile')
+            if not held_out and profile is not None and profile not in control_profiles:
+                control_profiles.append(profile)
+            stats.update({'partition': 'heldOut' if held_out else 'training', 'sampledWindows': len(samples)})
+            dataset_stats.append(stats)
+            emit('dataset', sessionId=session_id, windows=seen, sampled=len(samples), diagnostics=stats)
+        except (ValueError, KeyError, TypeError, OSError, EOFError, IndexError, OverflowError, av.error.FFmpegError) as error:
+            rejected[session_id] = str(error)[:500]
+            emit('skipped', sessionId=session_id, reason=rejected[session_id])
+    if len(pool) < config['minimumSamples'] or not any(sample[2].any() for sample in pool) or cancelled():
+        emit('complete', trained=False, accepted=[], rejected=rejected, samples=len(pool),
+             diagnostics={'dataset': dataset_stats, 'decodeSeconds': time.monotonic() - decode_started},
+             reason='Stopped/time limit' if cancelled() else 'Waiting for enough training-partition samples with human actions')
+        return
+
+    decode_seconds = time.monotonic() - decode_started
+    audit_started = time.monotonic()
+    numeric_mask, positive_weight = loss_setup(pool, schema)
+    training_rows = [row for row in dataset_stats if row.get('partition') == 'training']
+    natural_count = sum(row.get('usableWindows', 0) for row in training_rows)
+    frequencies = np.sum([row.get('commandEventCounts', [0] * SLOTS) for row in training_rows], axis=0) / max(1, natural_count)
+    audit_subset = pool[::max(1, len(pool) // 96)][:96]
+    before_training = evaluate(model, audit_subset, schema, job['actionThreshold'], frequencies, config['batchSize'], cancelled)
+    before_audit_seconds = time.monotonic() - audit_started
+    train_started = time.monotonic()
+    gradient_norms = []
+    losses = []
+    model.train()
+    for _ in range(config['passesPerJob']):
+        random.shuffle(pool)
+        for offset in range(0, len(pool), config['batchSize']):
+            if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
+                break
+            batch = pool[offset:offset + config['batchSize']]
+            images, states, targets, values, available = tensors(batch)
+            optimizer.zero_grad(set_to_none=True)
+            action_logits, predictions = model(images, states)
+            event_loss, value_loss = policy_losses(action_logits, predictions, targets, values, available, numeric_mask, positive_weight)
+            loss = event_loss + value_loss
+            if not torch.isfinite(loss):
+                raise ValueError('Nonfinite training loss; checkpoint not saved')
+            loss.backward()
+            gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
+            if not torch.isfinite(gradient_norm):
+                raise ValueError('Nonfinite training gradient; checkpoint not saved')
+            gradient_norms.append(float(gradient_norm))
+            optimizer.step()
+            steps += 1
+            examples += len(batch)
+            losses.append(loss.item())
+            if steps % 10 == 0:
+                emit('progress', steps=steps, examples=examples, loss=loss.item(),
+                     losses={'events': event_loss.item(), 'commandValues': value_loss.item()})
+        if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
+            break
+    if steps == initial_steps:
+        emit('complete', trained=False, accepted=[], rejected=rejected, reason='No training steps completed')
+        return
+
+    training_seconds = time.monotonic() - train_started
+    evaluation_started = time.monotonic()
+    distinct_windows.update(observed_windows)
+    unique_count = sum(distinct_windows.values())
+    coverage = [row for row in dataset_stats if row.get('partition') == 'heldOut']
+    unsupported_events = sum(row.get('unsupportedCommandEvents', 0) for row in coverage)
+    unsupported_windows = sum(row.get('unsupportedWindows', 0) for row in coverage)
+    evaluation = {'status': 'No supported held-out windows' if unsupported_events and not validation else 'waiting for held-out sessions',
+                  'windows': len(validation), 'unsupportedCommandEvents': unsupported_events,
+                  'unsupportedWindows': unsupported_windows}
+    if validation and not cancelled():
+        measured = evaluate(model, validation, schema, job['actionThreshold'], frequencies, config['batchSize'], cancelled)
+        evaluated = measured['windows']
+        precision, recall = measured.get('precision', 0), measured.get('recall', 0)
+        actual_positive = measured.get('positiveActions', 0)
+        mse = measured.get('numericMse', 0)
+        stop_actual = measured.get('stopEvents', 0)
+        stop_recall = measured.get('stopRecall', 0)
+        drive_actual = measured.get('movementEvents', 0)
+        drive_recall = measured.get('movementRecall', 0)
+        frequency_f1 = max((row['f1'] for row in measured.get('baselines', {}).get('trainingFrequency', [])), default=0)
+        held_out_sessions = sum(row.get('partition') == 'heldOut' and not row.get('skipReason') for row in dataset_stats)
+        gate_reasons = []
+        for condition, reason in [
+            (unsupported_events == 0, 'Held-out controls are not fully represented; see unsupportedCommands'),
+            (evaluated == len(validation), 'Evaluation interrupted'),
+            (held_out_sessions >= 2, 'Need two usable held-out recordings'),
+            (evaluated >= 128, 'Need 128 held-out windows'),
+            (actual_positive >= 32, 'Need 32 held-out command events'),
+            (stop_actual >= 8, 'Need eight held-out stop events'),
+            (drive_actual >= 16, 'Need 16 held-out movement commands'),
+            (drive_recall >= .5, 'Movement command recall below 0.5'),
+            (precision >= .5, 'Command precision below 0.5'),
+            (recall >= .5, 'Command recall below 0.5'),
+            (stop_recall >= .5, 'Stop recall below 0.5'),
+            (measured.get('f1', 0) > frequency_f1, 'Command F1 does not beat the training-frequency baseline'),
+            (mse <= .1, 'Command parameter error above 0.1'),
+        ]:
+            if not condition:
+                gate_reasons.append(reason)
+        evaluation = {**measured, 'status': 'imitation gate not met' if gate_reasons else 'passed imitation gate',
+                      'unsupportedCommandEvents': unsupported_events, 'unsupportedWindows': unsupported_windows,
+                      'method': 'Isolated held-out 100ms events; live selection, no temporal latch/physical rollout',
+                      'threshold': job['actionThreshold'], 'heldOutSessions': held_out_sessions,
+                      'gateReasons': gate_reasons}
+    after_training = evaluate(model, audit_subset, schema, job['actionThreshold'], frequencies, config['batchSize'], cancelled)
+    probe_seconds = 0
+    if not cancelled() and (learning_probe is None or steps - last_probe_steps >= 1000):
+        remaining = config['maxJobSeconds'] - (time.monotonic() - started)
+        if remaining > 10:
+            probe_started = time.monotonic()
+            learning_probe = probe(model, audit_subset, schema, job['actionThreshold'], frequencies,
+                                   cancelled, max_seconds=min(20, remaining / 2))
+            learning_probe['atTrainingStep'] = steps
+            last_probe_steps = steps
+            probe_seconds = time.monotonic() - probe_started
+    training_audit = {'before': before_training, 'after': after_training,
+                      'gradientNormBeforeClipping': {'min': min(gradient_norms), 'max': max(gradient_norms)},
+                      'positiveWeights': positive_weight[:len(schema)].tolist(),
+                      'naturalTrainingCommandFrequency': frequencies[:len(schema)].tolist(),
+                      'sampledCommandFrequency': np.mean([row[2] for row in pool], axis=0)[:len(schema)].tolist(),
+                      'learningProbe': learning_probe}
+    evaluation_seconds = time.monotonic() - evaluation_started - probe_seconds
+    publication = {'distinctMinutes': unique_count / 600,
+                   'newMinutes': (unique_count - published_windows) / 600,
+                   'minimumMinutes': config['minimumDrivingMinutes'],
+                   'requiredNewMinutes': config['newDrivingMinutesPerModel']}
+    reasons = []
+    if publication['distinctMinutes'] < config['minimumDrivingMinutes']:
+        reasons.append('Collecting distinct driving footage')
+    if publication['newMinutes'] < config['newDrivingMinutesPerModel']:
+        reasons.append('Waiting for new driving footage since last publication')
+    if steps - last_published < config['checkpointEverySteps']:
+        reasons.append('Training steps below publication interval')
+    if evaluation['status'] != 'passed imitation gate':
+        reasons.append(evaluation['status'])
+    if cancelled():
+        reasons.append('Job time budget reached')
+    publication['reason'] = '; '.join(reasons) or 'Latest updated'
+    save_started = time.monotonic()
+    published = None
+    # Every completed job exports an immutable candidate; Node owns promotion and retention.
+    adjectives = ['amber', 'curious', 'gentle', 'quiet', 'silver', 'bright', 'merry', 'sleepy']
+    animals = ['otter', 'finch', 'badger', 'robin', 'fox', 'heron', 'marten', 'wren']
+    model_id = str(uuid.uuid4())
+    name = f'{random.choice(adjectives)}-{random.choice(animals)}-{model_id[:8]}'
+    checkpoint_root = root / 'training' / 'checkpoints'
+    checkpoint_root.mkdir(exist_ok=True)
+    staging = checkpoint_root / f'.{model_id}.pending'
+    staging.mkdir()
+    weights_path = staging / 'weights.pt'
+    atomic_torch_save(torch, model.state_dict(), weights_path)
+    published = {
+        'id': model_id, 'name': name, 'createdAt': int(time.time() * 1000),
+        'trainingSteps': steps, 'examplesProcessed': examples,
+        'parameters': sum(p.numel() for p in model.parameters()),
+        'weightBytes': weights_path.stat().st_size,
+        'sha256': hashlib.sha256(weights_path.read_bytes()).hexdigest(),
+        'specification': specification(schema), 'trainedRovers': sorted(trained_rovers),
+        'controlProfiles': control_profiles,
+        'sourceSessions': accepted, 'trainingLoss': sum(losses) / len(losses),
+        'evaluation': evaluation['status'], 'validation': evaluation, 'distinctMinutes': publication['distinctMinutes'], 'dtype': 'float32', 'torchVersion': str(torch.__version__),
+    }
+    (staging / 'model.json').write_text(json.dumps(published, indent=2, allow_nan=False))
+    # A catalog reader sees both weights and metadata, or neither. Existing
+    # versions are never overwritten or deleted by the trainer.
+    staging.rename(checkpoint_root / model_id)
+    if not reasons:
+        last_published = steps
+        published_windows = unique_count
+    atomic_torch_save(torch, {
+        'version': VERSION, 'weights': model.state_dict(), 'optimizer': optimizer.state_dict(),
+        'schema': schema, 'steps': steps, 'examples': examples,
+        'lastPublished': last_published, 'trainedRovers': sorted(trained_rovers),
+        'controlProfiles': control_profiles, 'distinctWindows': distinct_windows,
+        'publishedWindows': published_windows, 'lastProbeSteps': last_probe_steps, 'learningProbe': learning_probe,
+    }, resume_path)
+    diagnostics = {'dataset': dataset_stats, 'trainingAudit': training_audit, 'probeSeconds': probe_seconds, 'beforeAuditSeconds': before_audit_seconds, 'decodeSeconds': decode_seconds,
+                   'trainingSeconds': training_seconds, 'evaluationSeconds': evaluation_seconds, 'checkpointSeconds': time.monotonic() - save_started,
+                   'cpuSeconds': time.process_time() - cpu_started,
+                   'peakRssMiB': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
+                   'dependencies': {'torch': str(torch.__version__), 'numpy': str(np.__version__), 'av': str(av.__version__)},
+                   'policy': specification(schema), 'browserVideoLatencyMs': None}
+    emit('complete', trained=True, accepted=accepted, rejected=rejected,
+         steps=steps, examples=examples, loss=sum(losses) / len(losses),
+         published=published, promote=not reasons, publication=publication, evaluation=evaluation, diagnostics=diagnostics, elapsedSeconds=time.monotonic() - started)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as error:
+        emit('error', message=str(error))
+        raise SystemExit(1)

server/src/services/turnService/index.js

@@ -397,6 +397,7 @@ function advanceTurn(roverId) {
     return;
   }
   if (queue.queue.length === 0) {
+    queue.current = null;
     stopRover(roverId);
     clearTurnTimer(roverId, queue);
     setActiveDriver(roverId, null);

webui/package.json

@@ -4,7 +4,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "dev": "VITE_ROVERD_URL=http://192.168.0.86:8080 vite",
+    "dev": "VITE_ROVERD_URL=https://rover.otter.land vite",
     "build": "vite build",
     "lint": "eslint .",
     "test": "node --test src/controls/inputs/*.test.js src/components/GamepadMappingSettings/*.test.js",

webui/src/components/HomeAssistantActivitiesPanel/index.jsx

@@ -24,8 +24,9 @@ function ActivityTile({ entity, connected, allowed, admin }) {
   // Translucent fills preserve text contrast; an amber border still identifies
   // locked tiles regardless of the admin's chosen color.
   const color = /^#[0-9a-f]{6}$/i.test(entity.color) ? entity.color : '#3b82f6';
-  const Control = controls[entity.type] || ReadOnlyControl;
-  const disabled = !connected || !entity.available || !allowed || (entity.locked && !admin);
+  // Ordinary users see only the value; admins retain the entity's actual input.
+  const Control = entity.readOnly && !admin ? ReadOnlyControl : controls[entity.type] || ReadOnlyControl;
+  const disabled = !connected || !entity.available || !allowed || ((entity.locked || entity.readOnly) && !admin);
   return <div className="flex min-w-0 flex-col gap-0.5 rounded-sm border px-0.5 py-0.5"
     style={{ borderColor: entity.locked ? '#b45309' : `${color}aa`, backgroundColor: `${color}33` }}>
     <div className="-mx-0.5 -mt-0.5 flex min-h-5 min-w-0 items-center gap-0.5 rounded-t px-0.5 py-0.5 text-white" style={{ backgroundColor: `${color}66` }}>

webui/src/components/VipPanel/index.jsx

@@ -11,6 +11,7 @@ import VipVerificationCard from '../vip/VipVerificationCard.jsx';
 import VipIdentityCard from '../vip/VipIdentityCard.jsx';
 import VipPrivateRoverAccessCard from '../vip/VipPrivateRoverAccessCard.jsx';
 import VipProfileImageCard from '../vip/VipProfileImageCard.jsx';
+import VipRoverLearningCard from '../vip/VipRoverLearningCard.jsx';
 
 export default function VipPanel() {
   const session = useSessionSelector((state) => state.session);
@@ -71,6 +72,7 @@ export default function VipPanel() {
         <div className="@[32rem]:col-span-2">
           {isVerified ? (
             <div className="space-y-2">
+              <VipRoverLearningCard />
               <VipMidiBeeperCard />
               <VipAudioUploadCard
                 ownRoverId={ownRoverId}

webui/src/components/vip/VipRoverLearningCard.jsx

@@ -0,0 +1,277 @@
+// One service snapshot supplies this entire card. Favorites never leave settings.
+import { useEffect, useState } from 'react';
+import { useSessionSelector } from '../../context/SessionContext.jsx';
+import { isFeatureEnabled } from '../../lib/features.js';
+import { useSocket } from '../../context/SocketContext.jsx';
+import { useSettingsNamespace } from '../../settings/index.js';
+import CardFrame from '../CardFrame/index.jsx';
+import { fieldClass } from './constants.js';
+
+const display = (value) => value == null ? 'Unavailable' : String(value);
+const bytes = (value) => value == null ? 'Unavailable' : `${(value / 1024 ** 2).toFixed(1)} MiB`;
+const time = (value) => value ? new Date(value).toLocaleString() : 'Unavailable';
+const age = (now, value) => value ? `${Math.max(0, (now - value) / 1000).toFixed(1)} s` : 'Unavailable';
+
+function Row({ label, children }) {
+  return <div className="flex flex-wrap items-baseline justify-between gap-x-2 gap-y-0.5 py-0.5">
+    <dt className="text-slate-400">{label}</dt>
+    <dd className="min-w-0 break-words text-right text-slate-200">{children}</dd>
+  </div>;
+}
+
+function Group({ title, children }) {
+  return <section className="surface-muted min-w-0 space-y-1">
+    <h4 className="font-semibold text-slate-200">{title}</h4>{children}
+  </section>;
+}
+
+const actionNames = { drive: 'Drive', motors: 'Brushes / vacuum', servo: 'Camera tilt', headlight: 'Headlight',
+  laser: 'Laser', horn: 'Horn', peripheral: 'Accessory', song: 'Beeper', raw: 'Recorded control' };
+
+function commandParameters(command) {
+  const fields = [];
+  const visit = (value, path) => {
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+      for (const [key, item] of Object.entries(value)) visit(item, path ? `${path}.${key}` : key);
+    } else {
+      fields.push(`${path}: ${typeof value === 'number' ? Number(value.toFixed(2)) : JSON.stringify(value)}`);
+    }
+  };
+  for (const [key, value] of Object.entries(command || {})) if (key !== 'type') visit(value, key);
+  return fields.join(' · ');
+}
+
+export default function VipRoverLearningCard() {
+  const enabled = useSessionSelector((state) => isFeatureEnabled(state, 'roverLearning') && Boolean(state.session?.isVerified));
+  const roverId = useSessionSelector((state) => state.session?.assignment?.roverId || null);
+  if (!enabled) return null;
+  return <LearningCard roverId={roverId} />;
+}
+
+function LearningCard({ roverId }) {
+  const socket = useSocket();
+  const [snapshot, setSnapshot] = useState(null);
+  const [selectedId, setSelectedId] = useState('');
+  const [detailsOpen, setDetailsOpen] = useState(false);
+  const [thresholdDraft, setThresholdDraft] = useState(null);
+  const [copyStatus, setCopyStatus] = useState('');
+  const [working, setWorking] = useState(false);
+  const [error, setError] = useState('');
+  const { value: settings, save } = useSettingsNamespace('roverLearning', { favorites: [] });
+  const favorites = Array.isArray(settings?.favorites) ? settings.favorites : [];
+
+  useEffect(() => {
+    let active = true;
+    const receive = (state) => { if (active) setSnapshot(state); };
+    const subscribe = () => {
+      if (!socket.connected) return;
+      socket.timeout(5000).emit('roverLearning:subscribe', { roverId, live: detailsOpen }, (failure, response) => {
+        if (!failure && response?.state) receive(response.state);
+      });
+    };
+    const disconnected = () => receive(null);
+    socket.on('roverLearning:state', receive);
+    socket.on('connect', subscribe);
+    socket.on('disconnect', disconnected);
+    subscribe();
+    // A config enable can mount this card before async service startup has
+    // attached handlers. Resubscribe periodically without another data API.
+    const timer = setInterval(subscribe, 10000);
+    return () => {
+      active = false;
+      clearInterval(timer);
+      socket.off('roverLearning:state', receive);
+      socket.off('connect', subscribe);
+      socket.off('disconnect', disconnected);
+      socket.emit('roverLearning:unsubscribe');
+    };
+  }, [socket, roverId, detailsOpen]);
+
+  const state = snapshot?.roverId === roverId ? snapshot : null;
+  const models = [...(state?.models || [])].sort((a, b) => Number(favorites.includes(b.id)) - Number(favorites.includes(a.id)) || Number(b.id === 'latest') - Number(a.id === 'latest') || b.createdAt - a.createdAt);
+  const chosen = models.find((model) => model.id === selectedId) || models.find((model) => model.compatible) || models[0];
+  const session = state?.session;
+  const training = state?.training;
+  const recording = state?.recording;
+  const limits = state?.limits;
+  const now = state?.updatedAt || 0;
+
+  const act = (action, value) => {
+    setWorking(true);
+    setError('');
+    socket.timeout(10000).emit('roverLearning:action', { action, value, roverId, modelId: chosen?.id }, (failure, response) => {
+      setWorking(false);
+      if (action === 'threshold') setThresholdDraft(null);
+      if (failure || response?.error) setError(response?.error || 'The server did not respond. Check the controller status before trying again.');
+      else if (response?.state) setSnapshot(response.state);
+    });
+  };
+  const commitThreshold = (event) => {
+    const value = Number(event.currentTarget.value);
+    if (!working && state?.control?.canAdjust && value !== session?.threshold) act('threshold', value);
+  };
+  const copyDiagnostics = async () => {
+    try {
+      if (!state?.diagnostics) return;
+      await navigator.clipboard.writeText(JSON.stringify(state.diagnostics));
+      setCopyStatus('Diagnostics copied');
+    } catch { setCopyStatus('Clipboard unavailable. Select and copy the report below.'); }
+  };
+  const favorite = () => {
+    if (!chosen) return;
+    save((current) => {
+      const ids = Array.isArray(current?.favorites) ? current.favorites : [];
+      return { ...current, favorites: ids.includes(chosen.id) ? ids.filter((id) => id !== chosen.id) : [...ids, chosen.id] };
+    });
+  };
+
+  return <CardFrame title="Rover learning" className="w-full" bodyClassName="text-sm text-slate-300">
+    <div className="space-y-2">
+      <p className="text-center text-xs text-slate-400">
+        {session ? `${session.modelName} · ${session.status}` : state ? 'Stopped' : 'Waiting for rover learning status…'}
+      </p>
+      <div className="flex flex-wrap items-center justify-center gap-1">
+        <select aria-label="Rover model" className={fieldClass} value={chosen?.id || ''} onChange={(event) => setSelectedId(event.target.value)} disabled={working || !models.length}>
+          {!models.length && <option value="">No published models yet</option>}
+          {models.map((model) => <option key={model.id} value={model.id}>{favorites.includes(model.id) ? '★ ' : ''}{model.name}{model.experimental ? ' (experimental)' : ''}{model.compatible ? '' : model.formatVersion !== 5 ? ' (older model format)' : ' (incompatible)'}</option>)}
+        </select>
+        <button type="button" className="button-dark text-sm disabled:opacity-50" aria-pressed={Boolean(chosen && favorites.includes(chosen.id))} disabled={!chosen} onClick={favorite}>
+          {chosen && favorites.includes(chosen.id) ? '★ Favorite' : '☆ Favorite'}
+        </button>
+      </div>
+      <div className="flex justify-center gap-1">
+        <button type="button" className="button-dark text-sm disabled:opacity-50" disabled={working || !state?.control?.canStart || !chosen?.compatible} onClick={() => act('start')}>Start</button>
+        <button type="button" className="button-dark text-sm disabled:opacity-50" disabled={working || !state?.control?.canStop} onClick={() => act('stop')}>Stop</button>
+      </div>
+      {state?.control?.canSnapshot && <div className="text-center">
+        <button type="button" className="button-dark text-sm disabled:opacity-50" disabled={working} onClick={() => act('snapshot')}>Save learner snapshot</button>
+      </div>}
+      {session && <section className="space-y-2" aria-label="Live model actions">
+        <p className="text-center text-xs text-sky-300">{session.predictionStatus || 'Waiting for fresh video and sensors'}</p>
+        <label className="surface-muted block space-y-1 text-xs">
+          <span className="flex justify-between gap-2"><span>Command threshold</span><span>{(thresholdDraft ?? session.threshold ?? .7).toFixed(2)}</span></span>
+          <input type="range" min="0.05" max="0.99" step="0.01" className="w-full accent-sky-400"
+            value={thresholdDraft ?? session.threshold ?? .7} disabled={working || !state?.control?.canAdjust}
+            onChange={(event) => setThresholdDraft(Number(event.target.value))}
+            onPointerUp={commitThreshold} onKeyUp={commitThreshold} onBlur={commitThreshold} />
+        </label>
+        <div className="grid max-h-52 grid-cols-1 gap-1 overflow-y-auto @[28rem]:grid-cols-2">
+          {session.proposals?.map((proposal) => {
+            const submission = proposal.reason === 'Proposed' && session.submissions?.find((item) => item.command.type === proposal.command.type
+              && (proposal.command.type !== 'peripheral' || (item.command.peripheral?.id === proposal.command.peripheral?.id
+                && item.command.peripheral?.control === proposal.command.peripheral?.control)));
+            const status = proposal.reason === 'Proposed' && submission ? submission.result : proposal.reason;
+            return <div key={proposal.slot} className="surface-muted space-y-1 text-xs">
+              <div className="flex justify-between gap-2"><span>{actionNames[proposal.command.type] || proposal.command.type}{proposal.command.type === 'peripheral' ? ` · ${proposal.command.peripheral?.id} / ${proposal.command.peripheral?.control}` : ''}</span><span>{proposal.score.toFixed(2)}</span></div>
+              <p className="break-words text-slate-300">{commandParameters(proposal.command)}</p>
+              <div role="meter" aria-label={`${actionNames[proposal.command.type] || proposal.command.type} score`}
+                aria-valuemin={0} aria-valuemax={1} aria-valuenow={proposal.score} className="relative h-2 overflow-hidden rounded bg-slate-800">
+                <div className={status === 'issued' ? 'h-full bg-emerald-400' : 'h-full bg-sky-500'} style={{ width: `${proposal.score * 100}%` }} />
+                <div className="absolute inset-y-0 w-px bg-white" style={{ left: `${(session.appliedThreshold ?? session.threshold ?? .7) * 100}%` }} />
+              </div>
+              <p className={status === 'issued' ? 'text-emerald-300' : 'text-slate-400'}>{status === 'issued' ? 'Submitted' : status}</p>
+              {submission && <p className="break-words text-slate-400">Sent: {commandParameters(submission.command)}</p>}
+            </div>;
+          })}
+        </div>
+      </section>}
+      {state?.control?.reason && <p className="text-center text-xs text-slate-400">{state.control.reason}</p>}
+      <p className="text-center text-xs text-slate-400">Recording: {recording ? recording.paused ? 'paused for disk space' : `${recording.sessions.filter((item) => item.video === 'Recording human driving').length} driving · ${recording.sessions.filter((item) => item.video === 'Dock lead-in buffer').length} dock buffer(s)` : 'unavailable'} · Training: {training?.status || 'unavailable'}</p>
+      {(error || state?.catalogError) && <p role="alert" className="break-words text-xs text-amber-300">{error || state.catalogError}</p>}
+      <details className="surface-muted text-xs" onToggle={(event) => setDetailsOpen(event.currentTarget.open)}>
+        <summary className="cursor-pointer py-1 font-semibold text-slate-200">System details</summary>
+        <div className="my-2 flex flex-wrap items-center gap-2">
+          <button type="button" className="button-dark disabled:opacity-50" disabled={!state?.diagnostics} onClick={copyDiagnostics}>Copy diagnostics</button>
+          <span role="status">{copyStatus}</span>
+        </div>
+        {copyStatus.startsWith('Clipboard unavailable') && <textarea readOnly aria-label="Diagnostics report" className="w-full" rows={4} value={JSON.stringify(state?.diagnostics || {})} onFocus={(event) => event.target.select()} />}
+        <div className="mt-2 grid min-w-0 grid-cols-1 gap-2 @[36rem]:grid-cols-2">
+          <Group title="Recording and storage"><dl>
+            <Row label="Recording size">{bytes(recording?.bytes)}</Row>
+            <Row label="Disk budget">{limits ? `${limits.recording.maxGiB} GiB` : 'Unavailable'}</Row>
+            <Row label="Free disk space">{bytes(recording?.freeBytes)}</Row>
+            <Row label="Minimum free space">{limits ? `${limits.recording.minimumFreeGiB} GiB` : 'Unavailable'}</Row>
+            <Row label="Session rotation">{limits ? `${limits.recording.sessionSeconds} s` : 'Unavailable'}</Row>
+            <Row label="Buffered events">{bytes(recording?.pendingBytes)}</Row>
+            <Row label="Dropped buffered events">{display(recording?.droppedPending)}</Row>
+          </dl>{recording?.sessions.map((item) => <div key={item.id} className="mt-1 border-t border-slate-700 pt-1">
+            <p>{item.roverId} · {item.video} · {age(now, item.startedAt)}</p>
+            <p>Dropped events: {item.droppedEvents}</p>{item.error && <p className="text-amber-300">{item.error}</p>}
+          </div>)}</Group>
+          <Group title="Dataset and training"><dl>
+            <Row label="Status">{training?.status || 'Unavailable'}</Row>
+            <Row label="Steps">{display(training?.steps)}</Row>
+            <Row label="Examples processed">{display(training?.examples)}</Row>
+            <Row label="Event / parameter losses">{training?.losses ? `${training.losses.events?.toFixed(4) ?? 'Unavailable'} / ${training.losses.commandValues?.toFixed(4) ?? 'Unavailable'}` : 'Unavailable'}</Row>
+            <Row label="Parameter error">{training?.evaluation?.numericMse?.toFixed(4) ?? 'Unavailable'}</Row>
+            <Row label="Missed / total stop events">{training?.evaluation?.stopEvents == null ? 'Unavailable' : `${training.evaluation.missedStops} / ${training.evaluation.stopEvents}`}</Row>
+            <Row label="Unwanted / no-command windows">{training?.evaluation?.noCommandWindows == null ? 'Unavailable' : `${training.evaluation.unwantedCommandWindows} / ${training.evaluation.noCommandWindows}`}</Row>
+            <Row label="Usable held-out recordings">{display(training?.evaluation?.heldOutSessions)}</Row>
+            <Row label="Training loss">{training?.loss == null ? 'Unavailable' : training.loss.toFixed(4)}</Row>
+            <Row label="Current job age">{training?.status === 'training' ? age(now, training.startedAt) : 'Not running'}</Row>
+            <Row label="Next attempt">{time(training?.nextAttemptAt)}</Row>
+            <Row label="Visited / skipped sessions">{training ? `${training.visitedSessions} / ${training.skippedSessions}` : 'Unavailable'}</Row>
+            <Row label="Sessions protected from retention">{display(training?.leasedSessions)}</Row>
+            <Row label="Last windows available / sampled">{training?.lastAvailableWindows == null ? 'Unavailable' : `${training.lastAvailableWindows} / ${training.lastSampledWindows}`}</Row>
+            <Row label="CPU threads / batch size">{limits ? `${limits.training.threads} / ${limits.training.batchSize}` : 'Unavailable'}</Row>
+            <Row label="Samples / passes per job">{limits ? `${limits.training.maxSamples} / ${limits.training.passesPerJob}` : 'Unavailable'}</Row>
+            <Row label="Distinct training minutes">{training?.publication?.distinctMinutes?.toFixed(1) ?? 'Unavailable'}</Row>
+            <Row label="Latest minimum / new minutes">{limits ? `${limits.training.minimumDrivingMinutes} / ${limits.training.newDrivingMinutesPerModel}` : 'Unavailable'}</Row>
+            <Row label="Latest update status">{training?.publication?.reason || 'Collecting data'}</Row>
+            <Row label="Held-out evaluation">{training?.evaluation?.status || 'Waiting for held-out data'}</Row>
+            <Row label="Held-out precision / recall">{training?.evaluation?.precision == null ? 'Unavailable' : `${training.evaluation.precision.toFixed(3)} / ${training.evaluation.recall.toFixed(3)}`}</Row>
+            <Row label="Evaluated training step">{display(training?.evaluatedSteps)}</Row>
+            <Row label="Mean / peak score on human commands">{training?.evaluation?.scoreOnHumanCommands?.mean == null ? 'Unavailable' : `${training.evaluation.scoreOnHumanCommands.mean.toFixed(3)} / ${training.evaluation.scoreOnHumanCommands.max.toFixed(3)}`}</Row>
+            <Row label="Training sample recall before / after">{training?.diagnostics?.trainingAudit?.after?.recall == null ? 'Unavailable' : `${training.diagnostics.trainingAudit.before.recall?.toFixed(3) ?? 'Unavailable'} / ${training.diagnostics.trainingAudit.after.recall.toFixed(3)}`}</Row>
+            <Row label="Memorization probe recall / steps">{training?.diagnostics?.trainingAudit?.learningProbe?.evaluation?.recall == null ? 'Unavailable' : `${training.diagnostics.trainingAudit.learningProbe.evaluation.recall.toFixed(3)} / ${training.diagnostics.trainingAudit.learningProbe.steps}`}</Row>
+            <Row label="Steps between Latest updates">{display(limits?.training.checkpointEverySteps)}</Row>
+          </dl>{training?.error && <p className="break-words text-amber-300">{training.error}</p>}
+            {training?.reason && <p>{training.reason}</p>}
+            {training?.evaluation?.gateReasons?.map((reason) => <p key={reason}>{reason}</p>)}
+            {training?.rejectionReasons?.map((reason) => <p className="break-words text-amber-300" key={reason}>{reason}</p>)}
+          </Group>
+          <Group title="Selected model"><dl>
+            <Row label="Name">{chosen?.name || 'None'}</Row>
+            <Row label="Created">{time(chosen?.createdAt)}</Row>
+            <Row label="Size">{bytes(chosen?.weightBytes)}</Row>
+            <Row label="Parameters">{chosen?.parameters?.toLocaleString() || 'Unavailable'}</Row>
+            <Row label="Steps / examples">{chosen ? `${chosen.trainingSteps} / ${chosen.examplesProcessed}` : 'Unavailable'}</Row>
+            <Row label="Control shapes">{display(chosen?.controls)}</Row>
+            <Row label="Evaluation">{chosen?.evaluation || 'Unavailable'}</Row>
+            <Row label="Distinct training minutes">{chosen?.distinctMinutes?.toFixed(1) ?? 'Unavailable'}</Row>
+            <Row label="Held-out precision / recall">{chosen?.validation?.precision == null ? 'Unavailable' : `${chosen.validation.precision.toFixed(3)} / ${chosen.validation.recall.toFixed(3)}`}</Row>
+            <Row label="Available models">{models.length}</Row>
+          </dl></Group>
+          <Group title="Live controller"><dl>
+            <Row label="Latest prediction">{session?.predictionStatus || 'Unavailable'}</Row>
+            <Row label="Action threshold">{display(session?.threshold)}</Row>
+            <Row label="State">{session?.status || 'Stopped'}</Row>
+            <Row label="Started by">{session?.startedBy || 'Unavailable'}</Row>
+            <Row label="Duration">{age(now, session?.startedAt)}</Row>
+            <Row label="Inference time">{session?.latencyMs == null ? 'Unavailable' : `${session.latencyMs.toFixed(1)} ms`}</Row>
+            <Row label="Video age">{age(now, session?.frameAt)}</Row>
+            <Row label="Sensor age">{age(now, session?.sensorAt)}</Row>
+            <Row label="Last command result">{session?.lastResult || 'Unavailable'}</Row>
+            <Row label="CPU threads">{display(limits?.driving.threads)}</Row>
+          </dl>{session?.lastAction && <pre className="mt-1 whitespace-pre-wrap break-all text-slate-400">{JSON.stringify(session.lastAction.command, null, 2)}</pre>}
+            {session?.proposals?.length > 0 && <div className="mt-2 max-h-72 space-y-1 overflow-auto">
+              {session.proposals.map((proposal) => <div key={proposal.slot} className="border-t border-slate-700 py-1">
+                <p>{proposal.command.type} · score {proposal.score.toFixed(3)} · {proposal.reason}</p>
+                <pre className="whitespace-pre-wrap break-all text-slate-400">{JSON.stringify(proposal.command)}</pre>
+              </div>)}
+            </div>}
+            {session?.submissions?.map((item, index) => <p key={index}>Submitted {item.command.type}: {item.result}</p>)}
+            {session?.stopReason && <p>{session.stopReason}</p>}
+          </Group>
+          <div className="@[36rem]:col-span-2"><Group title="Recent activity">
+            <ul className="max-h-56 space-y-1 overflow-y-auto">
+              {state?.activity?.map((entry, index) => <li key={`${entry.at}-${index}`} className="break-words">
+                <span className="text-slate-500">{time(entry.at)}</span> · {entry.message}{entry.roverId ? ` · ${entry.roverId}` : ''}{entry.detail ? ` · ${entry.detail}` : ''}
+              </li>)}
+            </ul>
+          </Group></div>
+        </div>
+      </details>
+    </div>
+  </CardFrame>;
+}