From 0ff03a74a5e113826bf0fc6d9399c45aec22e716 Mon Sep 17 00:00:00 2001 From: legop3 Date: Mon, 28 Sep 2026 00:29:21 -0400 Subject: [PATCH] slop --- Dockerfile | 10 + plans/rover learning service.md | 203 ++++++++++++-- server/src/services/commandService/index.js | 260 +++++++++--------- .../src/services/commandService/recording.js | 4 +- .../roverLearningService/capabilities.js | 15 + .../roverLearningService/configuration.js | 32 ++- .../services/roverLearningService/driving.js | 174 ++++++++++++ .../services/roverLearningService/models.js | 20 ++ .../roverLearningService/recording.js | 4 + .../services/roverLearningService/runtime.js | 58 +++- .../roverLearningService/socketGateway.js | 111 ++++++++ .../services/roverLearningService/storage.js | 2 +- .../services/roverLearningService/training.js | 177 ++++++++++++ .../roverLearningService/workers/dataset.py | 173 ++++++++++++ .../roverLearningService/workers/infer.py | 151 ++++++++++ .../roverLearningService/workers/policy.py | 131 +++++++++ .../workers/requirements.txt | 4 + .../roverLearningService/workers/train.py | 214 ++++++++++++++ webui/src/components/VipPanel/index.jsx | 4 + .../components/vip/VipRoverLearningCard.jsx | 175 ++++++++++++ 20 files changed, 1752 insertions(+), 170 deletions(-) create mode 100644 server/src/services/roverLearningService/capabilities.js create mode 100644 server/src/services/roverLearningService/driving.js create mode 100644 server/src/services/roverLearningService/models.js create mode 100644 server/src/services/roverLearningService/socketGateway.js create mode 100644 server/src/services/roverLearningService/training.js create mode 100644 server/src/services/roverLearningService/workers/dataset.py create mode 100644 server/src/services/roverLearningService/workers/infer.py create mode 100644 server/src/services/roverLearningService/workers/policy.py create mode 100644 server/src/services/roverLearningService/workers/requirements.txt create mode 100644 server/src/services/roverLearningService/workers/train.py create mode 100644 webui/src/components/vip/VipRoverLearningCard.jsx diff --git a/Dockerfile b/Dockerfile index cb7e133e..8a9fd5bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -97,6 +97,15 @@ RUN curl --fail --location --retry 3 \ && find /output/opt/roverd/googletts -type d -exec chmod 0755 {} + \ && find /output/opt/roverd/googletts -type f -exec chmod 0644 {} + +FROM architecture-check AS learning-dependencies +# CPU-only learning dependencies are built once. Disabled installations never +# launch Python or install packages as a side effect of starting Node. +RUN dnf install -y --setopt=install_weak_deps=False python3 python3-pip \ + && dnf clean all +COPY server/src/services/roverLearningService/workers/requirements.txt /tmp/learning-requirements.txt +RUN python3 -m venv /opt/rover-learning \ + && /opt/rover-learning/bin/pip install --no-cache-dir -r /tmp/learning-requirements.txt + FROM fedora:${FEDORA_VERSION} AS runtime ARG FEDORA_VERSION ARG TARGETARCH @@ -148,6 +157,7 @@ COPY --from=webui-build /build/server/public ./public COPY --from=native-workers /build/kinect/kinect_worker ./src/services/kinectService/native/kinect_worker COPY --from=native-workers /build/balance-board/balance_board_worker ./src/services/balanceBoardService/native/balance_board_worker COPY --from=packaged-tools /output/ / +COPY --from=learning-dependencies /opt/rover-learning /opt/rover-learning COPY --chmod=0755 server/bin/chromegtts-wav.py /usr/local/bin/chromegtts-wav COPY --chmod=0755 server/mediamtx/rover-snapshot-writer.sh /usr/local/bin/rover-snapshot-writer.sh diff --git a/plans/rover learning service.md b/plans/rover learning service.md index b9103b3b..2a5e05b1 100644 --- a/plans/rover learning service.md +++ b/plans/rover learning service.md @@ -2,11 +2,12 @@ ## Resume here -Status: implementation authorized; first recording/lifecycle increment is coded. -Training, model catalog, autonomous control, and the VIP card are not implemented. -The standalone -CPU benchmark exists at `perf/rover-model-benchmark.py` and was run successfully -on the user's actual server. No autonomous-driving capability is implemented. +Status: recording, bounded dataset/training, permanent named models, live inference, +checked autonomous command submission, and the verified VIP card are coded. +Human commands do not stop or pause autonomy (explicit latest user requirement). +Focused local checks passed; actual recording/training, live autonomous behavior, +concurrent server performance, and browser appearance remain unverified. +The standalone CPU benchmark was run successfully on the actual server. This document preserves the conversation's decisions across compactions and usage cutoffs. Keep the checklist and handoff section current during authorized @@ -19,12 +20,14 @@ AGENTS.md instructions: targeted investigation, minimal edits, no new tests unle requested, proportional direct verification, and no development processes left running. This development machine is not the actual rover server. -Next work: verify real recording output on the server, then define the actual -feature/action schema and implement training/checkpoint publication. The VIP entry -point is `webui/src/components/VipPanel/index.jsx`; verified cards are in its -`isVerified` branch, and client settings use `useSettingsNamespace`. The common -browser rover command sender is `webui/src/controls/commandPipeline.js`. Finish -the control-path inventory before declaring every capability learnable. +The user declined the manual server checklist and explicitly asked to continue. +Do not block further work or repeatedly ask them to do it. Preserve the unverified +status and have the reader reject unusable recordings automatically. + +Next work: verify the integrated feature against actual recordings and the real +camera/rover pipeline when that environment is available; inspect the VIP card in +the running browser. Do not invent successful driving results or block further +implementation on the previously declined manual checklist. ## Intended product @@ -101,13 +104,13 @@ the default development path. Intended layout: ```text rover-learning/ recordings/ Rolling video/events/session metadata (implemented) - models/ Permanent published models (pending) - training/ Resumable training/working state (pending) - catalog.sqlite Session/model/progress index (proposed; not created yet) + models/ Permanent UUID directories: weights.pt and model.json + training/ resume.pt, schedule.json, and current job.json ``` -Only `recordings/` currently exists when enabled. Its session manifests provide -the initial retention index; no placeholder model/training database is created. +Training/model directories are created only when service and training are enabled. +Session/model manifests and a small scheduling ledger are sufficient at present; +the previously proposed catalog.sqlite is not needed or created. Python compute and video processing must not block the Node event loop. Worker dependencies, deployment packaging, and process shutdown must be included in the implementation plan; installing Python packages at server startup is not assumed. @@ -166,8 +169,11 @@ an admin identity or send directly to the rover WebSocket. Recheck permissions server-side for actions and continued control. Losing the turn/assignment/connection or otherwise losing eligibility stops the autonomous session. A safe stop must still be possible after ordinary permission is revoked. -The exact policy for an incoming manual control command during autonomy is open; -immediate takeover is the proposed behavior. Prevent two controllers fighting. +Explicit user decision: incoming human commands do not stop or pause autonomy. +Both pass through the existing checked command path; subsequent model commands +can replace human commands. Stop remains explicit, with permission/disconnect, +stale input, and worker failure stops. Model-influenced sessions are excluded +from human demonstration recording, including manual inputs during autonomy. Model actions describe operating controls, not arbitrary server maintenance. Inventory reboot/update/raw commands and capability restrictions explicitly rather @@ -329,16 +335,21 @@ Published inference artifacts and resumable training state need not be identical - [x] Obtain IPI for feature implementation. - [ ] Inspect remaining UI/control paths and settle concrete contracts. - [x] Add service-owned config and dormant lifecycle integration. -- [ ] Add optional detailed command observer and shared checked submission path. +- [x] Add optional detailed command observer and shared checked submission path. - [ ] Implement timestamped recording, video worker ownership, and bounded dataset. Code is implemented for common commands, raw/decoded rover sensor frames, host stats, and existing rover state events. Live video/timing/retention verification and full control-path inventory remain; do not mark complete yet. -- [ ] Implement feature/action encoding and training worker with resource limits. -- [ ] Implement atomic checkpoint publication and permanent model catalog. -- [ ] Implement model inference, ownership, takeover, stops, and worker cleanup. -- [ ] Implement service socket gateway with complete per-user card state. -- [ ] Implement verified VIP card at top, model selection, local favorites, controls. +- [x] Implement feature/action encoding and training worker with resource limits. + Runtime import/dependency checks passed; training on actual recordings is still + unverified. The current schema limits and next inference work are described below. +- [x] Implement atomic checkpoint publication and permanent model catalog. + Code complete; no actual-data published checkpoint has been produced locally. +- [x] Implement model inference, ownership, explicit stops, and worker cleanup. + Human input does not stop autonomy; live execution remains unverified. +- [x] Implement service socket gateway with complete per-user card state. +- [x] Implement verified VIP card at top, model selection, local favorites, controls, + and expandable diagnostics. Browser appearance remains unverified. - [ ] Verify disabled startup and enable/disable cleanup directly. - [ ] Verify manual-control behavior remains intact and autonomous permission loss stops motion using an appropriate authorized runtime. @@ -420,8 +431,8 @@ Limits and follow-up: surviving children. Existing MediaMTX shutdown can exit the whole Node process before async metadata flush finishes; next enable marks those sessions interrupted. - Storage leases for concurrent training reads are pending with dataset sampling. -- Safe command submission extraction, autonomous provenance, training, permanent - catalog, model selection, client favorites, and UI remain pending. +- Safe command submission extraction, autonomous provenance, model selection, + client favorites, and UI remain pending. Training/catalog code follows below. Verification completed: @@ -438,7 +449,8 @@ Verification completed: - Local FFmpeg help confirms the RTSP transport and timeout options used. - No full build, full lint, new tests, live server startup, or rover hardware run. -Server verification needed for this increment: +Optional server verification reference (user declined doing this checklist; +do not require it before continuing implementation): 1. Deploy normally, initially disabled; confirm no `rover-learning` directory or learning FFmpeg workers are created by a fresh disabled startup. @@ -458,11 +470,142 @@ Server verification needed for this increment: 6. Verify rotation and retention with an intentionally chosen small recording budget, and confirm manual driving remains responsive during recording. -Return the session metadata, a small representative event sample (excluding any -unwanted personal text), FFprobe timestamps, and any `roverLearning` errors. Actual -server capture/timing and live cleanup remain unverified until these checks run. +Actual server capture/timing and live cleanup remain unverified. These steps are +a reference for future runtime investigation, not a pending request to the user. + +### Dataset/training/checkpoint increment + +The user supplied IPI, then asked to continue after a rate-limit interruption. +Authorization remains in effect. Implemented: + +- `workers/policy.py`: fixed-size spatial CNN/GRU with 64 action-shape slots and + up to 8 numeric output fields per shape. Four 160x120 RGB frames, spaced roughly + 300 ms apart, with 32 selected sensors plus missing-value masks and previous + command values/ages/known flags. About 805k parameters (larger than the benchmark). +- Numeric controls are normalized with `2/pi * atan(value/scale)`; the model + produces per-shape event logits and numeric values for the next 100 ms. + Command types supported: drive, motors, servo, headlight, laser, horn, peripheral, + song, raw. Strings/bools/arrays remain exact template constants; variable-size + songs/raw OI payloads are discrete demonstrations, not generated arbitrary bytes. + Maintenance commands, audio/text and sensor-stream setup are not action targets. + All recordings remain intact until normal dataset retention, regardless of + whether the trainer can use their commands. +- The schema grows only into unused fixed slots and is saved with each model. + Overflow (64 shapes/8 numeric fields/4096-byte commands) skips the affected + recording with a reason instead of silently truncating controls. This is a + concrete initial encoding limit, not a promise to represent unlimited hardware + or arbitrary song libraries. Revisit it if real recordings exhaust capacity. +- `workers/dataset.py`: bounded event loading (64 MiB/session), PyAV decoding, + absolute timestamp checks, causal sensor/action history, accepted-client-command + targets, and masks for observed action slots. Actual measured wheel speeds are + among the selected sensor inputs. Missing/stale sensors, video gaps, clock shifts, + incomplete recordings, data loss, and ambiguous conflicting actions are excluded. + Server-generated control overrides are history/boundaries, not demonstrations. +- Validation is automatic and cannot establish what frame a browser displayed. + A session is only accepted after its video finishes decoding; samples from a + subsequently corrupt session are discarded. Commands/sensors at future times + do not enter observation history. +- `workers/train.py`: finite CPU jobs, lower OS priority, shuffled bounded reservoir + sampling, sparse-event classification plus masked numeric regression, gradient + clipping, finite-loss checks, and AdamW resume state. Never sends rover commands. +- `training.js`: one owned worker, recording leases acquired within serialized + maintenance, bounded stdout/stderr, interval and wall-clock limits, shutdown, + persistent per-recording visit/rejection ledger, and progress state for the + future card. Retention protects leased recordings. Disk pressure cancels the + training job before files become eligible for pruning again. +- Atomic `training/resume.pt` includes model and optimizer state, schema, cumulative + steps/examples, and publication progress. Permanent model directories are + atomically renamed into visibility with weights and metadata together. +- `models.js`: reads the permanent metadata catalog. Each model has a UUID, two-word + generated name plus UUID suffix, creation time, steps/examples, schema, source + session IDs, trained rover IDs, training loss, parameter count, weight hash/size, + and `evaluation: unevaluated`. No model pruning, favorites, or server pin state. +- Dockerfile packages a CPU-only Python venv at `/opt/rover-learning` during image + build. Pinned requirements: torch 2.14.0+cpu, NumPy 2.5.3, PyAV 18.1.0. No runtime + downloads/installers. A disabled service does not load or run these dependencies. + +Training defaults beneath `roverLearning.training`: + +| Setting | Default | +|---|---:| +| enabled (still gated by outer service enabled=false) | true | +| threads | 4 | +| batchSize | 16 | +| maxSamples / minimumSamples | 512 / 128 | +| passesPerJob | 2 | +| roundsPerSession | 3 | +| maxStepsPerJob | 200 | +| checkpointEverySteps | 100 | +| intervalSeconds | 300 | +| maxJobSeconds | 600 | + +At most four completed retained sessions are selected per job. Samples are spread +over each full video via reservoir sampling, not always taken from its beginning. +The rounds limit is sampled training visits, not full passes over all recorded +frames. Once retained data reaches that limit, training waits for new sessions. +Candidate publication happens at job boundaries after enough accumulated steps; +it does not automatically deploy a model. No all-no-action sample pool is trained. + +Container rebuild is needed to include Python packages. For a non-container +deployment, install `workers/requirements.txt` into a dedicated venv and set +`training.python` to its Python path. The default path is container-specific. + +Verification performed in this increment: + +- Installed the actual pinned requirements in `/tmp/rover-bench-check-env`. +- Ran the production worker's `--check-dependencies`: PyTorch/PyAV/NumPy imports, + dataset imports, policy construction, and specification output succeeded. +- Four existing focused configuration tests passed (defaults, descriptions, + generated feature paths, normalization). The unrelated prior Discord default + assertion was not rerun or changed. +- Node syntax checks on scheduler/runtime/catalog passed. +- No new tests, fake recording fixtures, or throwaway test programs were created. +- No real recordings are available locally. Actual decoding/alignment, optimizer + execution against recordings, checkpoint publication/resume, and concurrent + server performance are not claimed verified. No full container build was run. + +The following controller/UI increment implements those remaining code paths. +Real-data and live-hardware verification still remain before claiming usable autonomy. The initial model is an experimental imitation policy, not a demonstrated +competent driver or a final architecture choice. When resuming after implementation begins, append concrete changed files, commands run/results, unresolved failures, decisions approved by the user, and the next bounded task here. Do not mark a checkbox complete merely because code exists; record verification limits and outstanding server/hardware work explicitly. + + +### Controller and VIP card increment + +- `commandService/index.js` exports the existing checked handler as `submitCommand`; + browser behavior remains in the same handler. Recording distinguishes model + requests from human requests without changing command payloads. +- `driving.js` owns per-rover CPU workers under the initiating live socket's + permissions. Explicit Stop, permission/connection loss, stale predictions, or + worker failure stop the session and send zero drive/motors and horn stop. + Human commands deliberately neither stop nor pause the model. +- `workers/infer.py` consumes live RTSP/TCP video plus sensor/command observations, + uses the training history format, validates weights/specification, resolves + competing command families, and latches discrete actions. It never writes to + the rover directly. Driving defaults: four CPU threads, 0.7 action threshold, + 2000ms stale limit, with a 30-second worker startup allowance. +- `capabilities.js` matches recorded hardware control profiles against the target + rover. Recordings, resumable training state, and published manifests preserve + those profiles; models without a matching profile are not selectable for Start. +- `runtime.js` excludes autonomous sessions from human demonstrations, manages + controller shutdown, and supplies bounded recent activity and storage state. +- `socketGateway.js` emits one complete authorized card object every second to + subscribers, refreshes the permanent catalog every 15 seconds, and accepts + checked Start/Stop actions. No server-side favorites are stored. +- `VipRoverLearningCard.jsx` is at the top of the verified VIP tab, gated by the + service feature flag. It uses existing CardFrame, field/button/surface styles; + favorites use browser settings. Its expandable section includes storage, + recording sessions, dataset rejection reasons, training progress/resource + limits, model size/loss/control count, inference timing, last action, and events. +- Verification: focused ESLint on the new card and VipPanel passed; four existing + configuration tests passed. The live worker's actual `--help` entry point loaded + PyTorch/PyAV/NumPy successfully. Shared-handler diff reviewed with whitespace + ignored: extraction preserves the previous command handler logic. +- No real dataset, checkpoint, camera stream, or rover was available locally; + end-to-end inference/training and browser rendering remain unverified. No new + tests or background processes were added for verification, and no full build + or full test suite was run. diff --git a/server/src/services/commandService/index.js b/server/src/services/commandService/index.js index aec12caa..0a896f89 100644 --- a/server/src/services/commandService/index.js +++ b/server/src/services/commandService/index.js @@ -259,144 +259,146 @@ module.exports = { setDriveCooldown, commandEvents, subscribeCommandRecording, + submitCommand, }; -io.on('connection', (socket) => { - function handleCommand({ roverId, type, data } = {}, cb) { - const reply = typeof cb === 'function' ? cb : () => {}; - try { - if (!roverId) { - throw new Error('roverId required'); +// Shared authorization/safety boundary for browser and service-owned control. +function submitCommand(socket, { roverId, type, data } = {}, cb) { + const reply = typeof cb === 'function' ? cb : () => {}; + try { + if (!roverId) { + throw new Error('roverId required'); + } + if (!type) { + throw new Error('type required'); + } + if (type === 'audioLevels') { + throw new Error('audioLevels command is service-managed'); + } + let payload = data ? { ...data } : {}; + if (type === 'headlight' && isHeadlightBlocked()) { + logger.info('Ignoring headlight command while darkness lock is active', { socketId: socket.id, roverId }); + reply({ ignored: true, reason: 'darknessActive' }); + return; + } + if (type === 'laser' && isLaserCommandBlockedByRoomLightLock(payload)) { + logger.info('Ignoring laser command while room lights are locked on', { + socketId: socket.id, + roverId, + action: getLaserAction(payload), + }); + reply({ ignored: true, reason: 'roomLightsLockedOn' }); + return; + } + const isRebootCommand = type === 'reboot'; + const isUpdateCommand = type === 'update'; + const isSongCommand = type === 'song' || (type === 'raw' && isSongRawPayload(payload)); + const isAdminSocket = isAdmin(socket); + if (!isAdminSocket && isDeterred(socket)) { + throw new Error('Not authorized'); + } + /* + Both structured song commands and raw Open Interface song payloads + reach this shared flag. Enforcing mute here covers the VIP MIDI beeper + and any future browser beeper without affecting unrelated driving. + */ + if (!isAdminSocket && isSongCommand && isMuted(socket)) { + throw new Error('Muted'); + } + // Rover updates run a privileged, root-owned helper on the Pi. Keep this + // in the same explicit admin-only branch as reboot instead of relying on + // drive ownership checks, because having a turn should not grant system + // maintenance privileges. + if ((isRebootCommand || isUpdateCommand) && !isAdminSocket) { + throw new Error('Not authorized'); + } + if (!isSongCommand && !isRebootCommand && !isUpdateCommand && !roverManager.canDrive(roverId, socket)) { + throw new Error('Not your turn or no control'); + } + const driveDirect = payload?.driveDirect; + if (type === 'drive' && driveDirect) { + if (!isAdminSocket) { + const safeDrive = roverManager.applyPrivateDriveSafety(roverId, socket, driveDirect); + if (safeDrive) { + payload.driveDirect = safeDrive; + } } - if (!type) { - throw new Error('type required'); - } - if (type === 'audioLevels') { - throw new Error('audioLevels command is service-managed'); - } - let payload = data ? { ...data } : {}; - if (type === 'headlight' && isHeadlightBlocked()) { - logger.info('Ignoring headlight command while darkness lock is active', { socketId: socket.id, roverId }); - reply({ ignored: true, reason: 'darknessActive' }); - return; + const left = Number(payload?.driveDirect?.left); + const right = Number(payload?.driveDirect?.right); + const speed = Math.max(Math.abs(left), Math.abs(right)); + if (!isAdminSocket) { + const blockedUntil = driveCooldowns.get(roverId); + if (blockedUntil && Date.now() < blockedUntil && speed > 0) { + const reason = isLockdownAdmin(socket) + ? 'Drive blocked: cooldown' + : 'Drive blocked: safety cooldown'; + throw new Error(reason); + } } - if (type === 'laser' && isLaserCommandBlockedByRoomLightLock(payload)) { - logger.info('Ignoring laser command while room lights are locked on', { + if (speed > 0) { + let direction = 'turn'; + if (left > 0 && right > 0) direction = 'forward'; + if (left < 0 && right < 0) direction = 'backward'; + lastDriveActivity.set(roverId, { + ts: Date.now(), socketId: socket.id, - roverId, - action: getLaserAction(payload), - }); - reply({ ignored: true, reason: 'roomLightsLockedOn' }); - return; - } - const isRebootCommand = type === 'reboot'; - const isUpdateCommand = type === 'update'; - const isSongCommand = type === 'song' || (type === 'raw' && isSongRawPayload(payload)); - const isAdminSocket = isAdmin(socket); - if (!isAdminSocket && isDeterred(socket)) { - throw new Error('Not authorized'); - } - /* - Both structured song commands and raw Open Interface song payloads - reach this shared flag. Enforcing mute here covers the VIP MIDI beeper - and any future browser beeper without affecting unrelated driving. - */ - if (!isAdminSocket && isSongCommand && isMuted(socket)) { - throw new Error('Muted'); - } - // Rover updates run a privileged, root-owned helper on the Pi. Keep this - // in the same explicit admin-only branch as reboot instead of relying on - // drive ownership checks, because having a turn should not grant system - // maintenance privileges. - if ((isRebootCommand || isUpdateCommand) && !isAdminSocket) { - throw new Error('Not authorized'); - } - if (!isSongCommand && !isRebootCommand && !isUpdateCommand && !roverManager.canDrive(roverId, socket)) { - throw new Error('Not your turn or no control'); - } - const driveDirect = payload?.driveDirect; - if (type === 'drive' && driveDirect) { - if (!isAdminSocket) { - const safeDrive = roverManager.applyPrivateDriveSafety(roverId, socket, driveDirect); - if (safeDrive) { - payload.driveDirect = safeDrive; - } - } - const left = Number(payload?.driveDirect?.left); - const right = Number(payload?.driveDirect?.right); - const speed = Math.max(Math.abs(left), Math.abs(right)); - if (!isAdminSocket) { - const blockedUntil = driveCooldowns.get(roverId); - if (blockedUntil && Date.now() < blockedUntil && speed > 0) { - const reason = isLockdownAdmin(socket) - ? 'Drive blocked: cooldown' - : 'Drive blocked: safety cooldown'; - throw new Error(reason); - } - } - if (speed > 0) { - let direction = 'turn'; - if (left > 0 && right > 0) direction = 'forward'; - if (left < 0 && right < 0) direction = 'backward'; - lastDriveActivity.set(roverId, { - ts: Date.now(), - socketId: socket.id, - direction, - speed, - isAdmin: isAdminSocket, - }); - } - } - - if (type === 'drive' || type === 'motors') { - /* - Role is supplied at the command boundary because telemetry does not - identify the operator who produced the active motor intent. Admin and - lockdown commands therefore enter the service explicitly bypassed; - they are recorded for status visibility but are never scaled, blocked, - or countermanded by a later sensor frame. - */ - payload = overcurrentProtectionService.protectCommand(roverId, type, payload, { - bypassed: isAdminSocket, + direction, + speed, + isAdmin: isAdminSocket, }); } - const id = issueCommand(roverId, { type, ...payload }); - commandEvents.emit('observation', { - ts: Date.now(), - roverId: String(roverId), - type, - commandId: id, - outcome: 'issued', - socketId: socket.id, - // Payloads are omitted deliberately: raw OI, TTS, and maintenance - // commands can carry arbitrary content. Their structured type/outcome - // supplies analytics without accidentally persisting secret material. - }); - logger.info('Queued command', socket.id, roverId, type); - if (shouldRecordTurnActivity(type, payload)) { - try { - const { recordActivity } = require('../turnService'); - recordActivity(roverId, socket.id); - } catch (err) { - // Activity recording is best-effort because command delivery should not fail if turn bookkeeping has a transient issue. - } - } - reply({ id }); - } catch (err) { - logger.warn('Command rejected', socket.id, err.message); - commandEvents.emit('observation', { - ts: Date.now(), - roverId: roverId ? String(roverId) : null, - type: type || 'unknown', - outcome: 'rejected', - socketId: socket.id, - error: err.message, - }); - reply({ error: err.message }); } - } - const recordedCommand = (request, reply) => recordCommandRequest(socket, request, reply, handleCommand); + if (type === 'drive' || type === 'motors') { + /* + Role is supplied at the command boundary because telemetry does not + identify the operator who produced the active motor intent. Admin and + lockdown commands therefore enter the service explicitly bypassed; + they are recorded for status visibility but are never scaled, blocked, + or countermanded by a later sensor frame. + */ + payload = overcurrentProtectionService.protectCommand(roverId, type, payload, { + bypassed: isAdminSocket, + }); + } + const id = issueCommand(roverId, { type, ...payload }); + commandEvents.emit('observation', { + ts: Date.now(), + roverId: String(roverId), + type, + commandId: id, + outcome: 'issued', + socketId: socket.id, + // Payloads are omitted deliberately: raw OI, TTS, and maintenance + // commands can carry arbitrary content. Their structured type/outcome + // supplies analytics without accidentally persisting secret material. + }); + logger.info('Queued command', socket.id, roverId, type); + if (shouldRecordTurnActivity(type, payload)) { + try { + const { recordActivity } = require('../turnService'); + recordActivity(roverId, socket.id); + } catch (err) { + // Activity recording is best-effort because command delivery should not fail if turn bookkeeping has a transient issue. + } + } + reply({ id }); + } catch (err) { + logger.warn('Command rejected', socket.id, err.message); + commandEvents.emit('observation', { + ts: Date.now(), + roverId: roverId ? String(roverId) : null, + type: type || 'unknown', + outcome: 'rejected', + socketId: socket.id, + error: err.message, + }); + reply({ error: err.message }); + } +} + +io.on('connection', (socket) => { + const recordedCommand = (request, reply) => recordCommandRequest(socket, request, reply, (message, cb) => submitCommand(socket, message, cb)); socket.on('command', recordedCommand); socket.on('command:issue', recordedCommand); diff --git a/server/src/services/commandService/recording.js b/server/src/services/commandService/recording.js index a97117a4..a556278e 100644 --- a/server/src/services/commandService/recording.js +++ b/server/src/services/commandService/recording.js @@ -33,9 +33,9 @@ function observeCommand(event) { } } -function recordCommandRequest(socket, request, reply, execute) { +function recordCommandRequest(socket, request, reply, execute, source = 'client') { if (!observers.size) return execute(request, reply); - const context = { requestId: randomUUID(), socketId: socket.id, userId: socket.data?.userId || null }; + const context = { requestId: randomUUID(), socketId: socket.id, userId: socket.data?.userId || null, source }; const scope = { active: true, context }; return requests.run(scope, () => { try { diff --git a/server/src/services/roverLearningService/capabilities.js b/server/src/services/roverLearningService/capabilities.js new file mode 100644 index 00000000..c5e96f49 --- /dev/null +++ b/server/src/services/roverLearningService/capabilities.js @@ -0,0 +1,15 @@ +// Match actuator configuration, not a rover's display name or changing telemetry. +const { isDeepStrictEqual } = require('node:util'); + +function controlProfile(meta = {}) { + return Object.fromEntries(['cameraServo', 'peripherals', 'headlight', 'laser', 'roomba', 'platform'] + .map((key) => [key, meta?.[key] ?? null])); +} + +function compatible(model, record) { + if (model?.specification?.version !== 1 || !record) return false; + const profile = controlProfile(record.meta); + return Array.isArray(model.controlProfiles) && model.controlProfiles.some((saved) => isDeepStrictEqual(saved, profile)); +} + +module.exports = { controlProfile, compatible }; diff --git a/server/src/services/roverLearningService/configuration.js b/server/src/services/roverLearningService/configuration.js index af0542b1..670bea68 100644 --- a/server/src/services/roverLearningService/configuration.js +++ b/server/src/services/roverLearningService/configuration.js @@ -1,5 +1,5 @@ // Metadata only: loading the configuration must never start the learning runtime. -const { strictObject, boolean, integer, number } = require('../../configuration/schemaHelpers'); +const { strictObject, boolean, integer, number, string } = require('../../configuration/schemaHelpers'); module.exports = { key: 'roverLearning', @@ -7,13 +7,39 @@ module.exports = { defaultValue: { enabled: false, recording: { maxGiB: 20, minimumFreeGiB: 2, sessionSeconds: 300 }, + driving: { threads: 4, actionThreshold: 0.7, staleMs: 2000 }, + training: { + enabled: true, python: '/opt/rover-learning/bin/python', threads: 4, + batchSize: 16, maxSamples: 512, minimumSamples: 128, passesPerJob: 2, + roundsPerSession: 3, maxStepsPerJob: 200, checkpointEverySteps: 100, + intervalSeconds: 300, maxJobSeconds: 600, + }, }, schema: strictObject({ - enabled: boolean({ description: 'Enable rover learning recording. Disabled starts no recording workers or storage. Training and autonomous control are not implemented yet.' }), + enabled: boolean({ description: 'Enable human-session recording, background learning, and verified-user model control. Disabled starts no workers or storage.' }), + driving: strictObject({ + threads: integer({ minimum: 1, maximum: 16, description: 'CPU threads per active model controller.' }), + actionThreshold: number({ minimum: 0.5, maximum: 0.99, description: 'Minimum predicted action probability for issuing a new control event.' }), + staleMs: integer({ minimum: 1000, maximum: 5000, description: 'Stop when video, sensors, or model predictions are older than this many milliseconds.' }), + }, { description: 'Live model execution. Manual inputs do not stop or pause the model.', required: ['threads', 'actionThreshold', 'staleMs'] }), recording: strictObject({ maxGiB: number({ minimum: 0.1, maximum: 100000, description: 'Rolling recording budget in GiB. Checked every 15 seconds; active files can temporarily exceed it. Published models are never pruned.' }), minimumFreeGiB: number({ minimum: 0.1, maximum: 100000, description: 'Pause recording below this amount of available filesystem space, in GiB.' }), sessionSeconds: integer({ minimum: 30, maximum: 3600, description: 'Rotate video and telemetry sessions after this many seconds so completed sessions can be pruned.' }), }, { description: 'Bounded recording storage. Limits never delete published models.', required: ['maxGiB', 'minimumFreeGiB', 'sessionSeconds'] }), - }, { description: 'Optional rover learning service. Currently provides passive recording only.', required: ['enabled', 'recording'] }), + training: strictObject({ + enabled: boolean({ description: 'Train periodically from completed usable human recordings while the service is enabled.' }), + python: string({ minLength: 1, description: 'Python executable with the service requirements installed. The container supplies /opt/rover-learning/bin/python.' }), + threads: integer({ minimum: 1, maximum: 32, description: 'CPU threads for the lower-priority training process.' }), + batchSize: integer({ minimum: 1, maximum: 64, description: 'Training windows per optimizer step.' }), + maxSamples: integer({ minimum: 128, maximum: 2048, description: 'Maximum decoded windows retained in RAM per job. Each contains four 160x120 RGB frames.' }), + minimumSamples: integer({ minimum: 1, maximum: 128, description: 'Usable sampled windows required before training.' }), + passesPerJob: integer({ minimum: 1, maximum: 10, description: 'Shuffled passes over the sampled windows per job.' }), + roundsPerSession: integer({ minimum: 1, maximum: 20, description: 'Maximum successful sampled training visits per retained recording; pauses repeated training on unchanged data.' }), + maxStepsPerJob: integer({ minimum: 1, maximum: 10000, description: 'Optimizer step cap per finite training job.' }), + checkpointEverySteps: integer({ minimum: 1, maximum: 100000, description: 'Accumulated training steps between permanent named model publications.' }), + intervalSeconds: integer({ minimum: 30, maximum: 86400, description: 'Minimum seconds between completed training jobs.' }), + maxJobSeconds: integer({ minimum: 30, maximum: 3600, description: 'Wall-clock budget including video decoding; slow/stuck workers are stopped.' }), + }, { description: 'Bounded CPU training, sampling, and permanent checkpoint publication.', required: ['enabled', 'python', 'threads', 'batchSize', 'maxSamples', 'minimumSamples', 'passesPerJob', 'roundsPerSession', 'maxStepsPerJob', 'checkpointEverySteps', 'intervalSeconds', 'maxJobSeconds'] }), + }, { description: 'Optional rover recording and learning service.', required: ['enabled', 'recording', 'training', 'driving'] }), }; diff --git a/server/src/services/roverLearningService/driving.js b/server/src/services/roverLearningService/driving.js new file mode 100644 index 00000000..e4302da0 --- /dev/null +++ b/server/src/services/roverLearningService/driving.js @@ -0,0 +1,174 @@ +// Autonomy owns a live user's control context, never an administrative identity. +const fs = require('node:fs/promises'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const roverManager = require('../roverManager'); +const { isVerified, isDeterred } = require('../verificationService'); +const { isAdmin } = require('../roleService'); +const { submitCommand, issueCommand, subscribeCommandRecording } = require('../commandService'); +const { recordCommandRequest } = require('../commandService/recording'); +const { compatible } = require('./capabilities'); + +const OPERATING = new Set(['drive', 'motors', 'servo', 'headlight', 'laser', 'horn', 'peripheral', 'song', 'raw']); +const clamp = (value, min, max) => Math.max(min, Math.min(max, Number(value) || 0)); + +function canControl(socket, roverId) { + return Boolean(socket?.connected && isVerified(socket) && (isAdmin(socket) || !isDeterred(socket)) + && roverManager.rovers.get(roverId)?.ws && roverManager.canDrive(roverId, socket)); +} + +function createDriving({ root, config, onChange, activity }) { + const sessions = new Map(); + let stopping = false; + let watchdog; + let unsubscribe; + + function sendState(session, event) { + const input = session.worker.stdin; + if (input.destroyed || !input.writable || input.writableLength > 256 * 1024) return; + input.write(JSON.stringify(event) + '\n'); + } + + function transmit(session, command) { + if (session.status === 'stopping') return; + if (!canControl(session.socket, session.roverId)) { stop(session.roverId, 'Control permission lost'); return; } + if (!OPERATING.has(command?.type)) throw new Error('Unsupported model command'); + const record = roverManager.rovers.get(session.roverId); + if (command.type === 'drive') { + command.driveDirect = { left: clamp(command.driveDirect?.left, -500, 500), right: clamp(command.driveDirect?.right, -500, 500) }; + } + if (command.type === 'motors') { + command.motorPwm = { main: clamp(command.motorPwm?.main, -127, 127), side: clamp(command.motorPwm?.side, -127, 127), vacuum: clamp(command.motorPwm?.vacuum, 0, 127) }; + } + if (command.type === 'servo') { + const servo = record.meta?.cameraServo; + if (!servo?.enabled) return; + command.servo.angle = clamp(command.servo?.angle, servo.minAngle ?? 0, servo.maxAngle ?? 180); + } + const { type, ...data } = command; + recordCommandRequest(session.socket, { roverId: session.roverId, type, data }, (result) => { + session.lastResult = result.error || result.reason || 'issued'; + if (result.id) session.lastAction = { at: Date.now(), command }; + }, (request, reply) => submitCommand(session.socket, request, reply), 'model'); + } + + function stop(roverId, reason = 'Stopped by user') { + const session = sessions.get(roverId); + if (!session) return Promise.resolve(); + if (session.done) return session.done; + session.status = 'stopping'; + session.stopReason = reason; + // A stop must still reach the rover after turn permission is revoked. + for (const command of [ + { type: 'drive', driveDirect: { left: 0, right: 0 } }, + { type: 'motors', motorPwm: { main: 0, side: 0, vacuum: 0 } }, + { type: 'horn', horn: { action: 'stop' } }, + ]) { try { issueCommand(roverId, command); } catch { /* Rover may already be offline. */ } } + session.worker.kill('SIGTERM'); + const timer = setTimeout(() => session.worker.kill('SIGKILL'), 2000); + session.done = session.closed.then(() => { + clearTimeout(timer); + sessions.delete(roverId); + activity('Controller stopped', { roverId, reason }); + onChange(); + }); + onChange(); + return session.done; + } + + async function start(socket, roverId, modelId) { + if (stopping || !canControl(socket, roverId)) throw new Error('You cannot control this rover'); + if (!/^[0-9a-f-]{36}$/.test(modelId)) throw new Error('Invalid model'); + const directory = path.join(root, 'models', modelId); + const metadata = JSON.parse(await fs.readFile(path.join(directory, 'model.json'), 'utf8')); + if (metadata.id !== modelId || !compatible(metadata, roverManager.rovers.get(roverId))) { + throw new Error('Model is not compatible with this rover'); + } + if (stopping || !canControl(socket, roverId)) throw new Error('Control permission changed'); + if (sessions.has(roverId)) throw new Error('Stop the current controller before starting another model'); + const worker = spawn(config.training.python, [path.join(__dirname, 'workers/infer.py'), '--model', directory, + '--url', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`, '--threads', String(config.driving.threads), + '--threshold', String(config.driving.actionThreshold)], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } }); + const session = { worker, socket, roverId, modelId, modelName: metadata.name, status: 'starting', startedAt: Date.now(), lastPredictionAt: null }; + session.closed = new Promise((resolve) => worker.once('close', resolve)); + sessions.set(roverId, session); + let output = ''; + let stderr = ''; + worker.stdin.on('error', () => { stop(roverId, 'Worker input closed'); }); + worker.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-1000); }); + worker.on('error', (error) => { stop(roverId, error.message); }); + worker.once('close', () => { stop(roverId, stderr || 'Worker exited'); }); + worker.stdout.on('data', (chunk) => { + if (session.status === 'stopping') return; + output += chunk; + if (output.length > 1024 * 1024) { stop(roverId, 'Worker output exceeded limit'); return; } + let newline; + while ((newline = output.indexOf('\n')) >= 0) { + const line = output.slice(0, newline); + output = output.slice(newline + 1); + try { + const message = JSON.parse(line); + if (message.kind === 'error') throw new Error(message.message); + if (message.kind !== 'prediction') continue; + if (!Number.isFinite(message.frameAt) || Math.abs(Date.now() - message.frameAt) > config.driving.staleMs) throw new Error('Video is stale'); + if (!Number.isFinite(message.sensorAt) || Math.abs(Date.now() - message.sensorAt) > config.driving.staleMs) throw new Error('Sensors are stale'); + if (!canControl(socket, roverId)) throw new Error('Control permission lost'); + session.status = 'running'; + session.lastPredictionAt = Date.now(); + session.frameAt = message.frameAt; + session.sensorAt = message.sensorAt; + session.latencyMs = message.latencyMs; + if (!Array.isArray(message.commands) || message.commands.length > 64) throw new Error('Invalid action batch'); + // Manual commands intentionally do not stop or pause this controller. + for (const command of message.commands) transmit(session, command); + } catch (error) { stop(roverId, error.message); break; } + } + }); + const sensors = roverManager.rovers.get(roverId)?.lastSensor?.decoded; + if (sensors) sendState(session, { kind: 'sensor', ts: Date.now(), data: sensors }); + activity('Controller started', { roverId, model: metadata.name, socketId: socket.id }); + onChange(); + } + + function sensor(event) { + const session = sessions.get(String(event.roverId)); + if (session) sendState(session, { kind: 'sensor', ts: Date.now(), data: event.sensors }); + } + + function attach() { + roverManager.managerEvents.on('sensor', sensor); + unsubscribe = subscribeCommandRecording((event) => { + const session = sessions.get(String(event.roverId)); + if (session && event.phase === 'dispatched') sendState(session, { kind: 'command', ts: event.ts, data: event.command }); + }); + watchdog = setInterval(() => { + for (const session of sessions.values()) { + if (!canControl(session.socket, session.roverId)) stop(session.roverId, 'Control permission lost'); + else if (Date.now() - (session.lastPredictionAt || session.startedAt) > (session.lastPredictionAt ? config.driving.staleMs : 30000)) { + stop(session.roverId, 'No fresh model predictions'); + } + } + }, 250); + } + + return { + attach, start, stop, canControl, + isActive: (id) => sessions.has(id), + getState: (id) => { + const session = sessions.get(id); + if (!session) return null; + const { worker, socket, closed, done, ...state } = session; + return { ...state, startedBy: socket.data?.nickname || socket.id, socketId: socket.id }; + }, + shutdown: async () => { + stopping = true; + clearInterval(watchdog); + unsubscribe?.(); + roverManager.managerEvents.off('sensor', sensor); + await Promise.all([...sessions.keys()].map((id) => stop(id, 'Service disabled'))); + }, + kill: () => { for (const session of sessions.values()) session.worker.kill('SIGKILL'); }, + }; +} + +module.exports = { createDriving, canControl }; diff --git a/server/src/services/roverLearningService/models.js b/server/src/services/roverLearningService/models.js new file mode 100644 index 00000000..630abbfa --- /dev/null +++ b/server/src/services/roverLearningService/models.js @@ -0,0 +1,20 @@ +// Published directories are immutable and become visible via atomic rename. +const fs = require('node:fs/promises'); +const path = require('node:path'); + +async function listModels(root) { + const directory = path.join(root, 'models'); + let entries; + try { entries = await fs.readdir(directory, { withFileTypes: true }); } + catch (error) { if (error.code === 'ENOENT') return []; throw error; } + const models = []; + for (const entry of entries) { + if (!entry.isDirectory() || !/^[0-9a-f-]{36}$/.test(entry.name)) continue; + const metadata = JSON.parse(await fs.readFile(path.join(directory, entry.name, 'model.json'), 'utf8')); + if (metadata.id !== entry.name) throw new Error(`Model identity mismatch: ${entry.name}`); + models.push(metadata); + } + return models.sort((a, b) => b.createdAt - a.createdAt); +} + +module.exports = { listModels }; diff --git a/server/src/services/roverLearningService/recording.js b/server/src/services/roverLearningService/recording.js index 74d5798e..45c4caaa 100644 --- a/server/src/services/roverLearningService/recording.js +++ b/server/src/services/roverLearningService/recording.js @@ -5,6 +5,7 @@ const fsp = require('node:fs/promises'); const path = require('node:path'); const { randomUUID } = require('node:crypto'); const { spawn } = require('node:child_process'); +const { controlProfile } = require('./capabilities'); const MAX_BUFFER_BYTES = 2 * 1024 * 1024; @@ -20,6 +21,7 @@ async function createRecording({ root, roverId, snapshot, logger }) { await fsp.mkdir(directory); const metadata = { version: 1, id, roverId, startedAt: Date.now(), endedAt: null, + controlProfile: controlProfile(snapshot.meta), timing: 'Video PTS uses server receiver Unix seconds. Events use Unix milliseconds and process monotonic nanoseconds. Browser display delay is not measured.', video: { file: 'video.mkv', codec: 'source-copy', timestampUnit: 'seconds', origin: 'unix' }, events: 'events.ndjson', droppedEvents: 0, @@ -111,6 +113,8 @@ async function createRecording({ root, roverId, snapshot, logger }) { id, directory, startedAt: metadata.startedAt, append, stop, get failed() { return Boolean(failure); }, get stopping() { return stopping; }, + getStatus: () => ({ id, roverId, startedAt: metadata.startedAt, droppedEvents: metadata.droppedEvents, + video: childClosed ? 'stopped' : 'recorder running', error: failure, stopping }), // Process exit is synchronous; normal disable uses stop() and awaits close. kill: () => { if (!childClosed) child.kill('SIGKILL'); }, }; diff --git a/server/src/services/roverLearningService/runtime.js b/server/src/services/roverLearningService/runtime.js index 28eab2a7..aca2a996 100644 --- a/server/src/services/roverLearningService/runtime.js +++ b/server/src/services/roverLearningService/runtime.js @@ -1,4 +1,4 @@ -// Enabled-only recording runtime. No training or driving worker is started here. +// Enabled-only recording, training, controller, and card lifecycle. const fs = require('node:fs/promises'); const path = require('node:path'); const { resolveDataPath } = require('../../helpers/dataPaths'); @@ -11,10 +11,27 @@ const { isDeterred, verificationEvents } = require('../verificationService'); const { subscribeCommandRecording, commandEvents } = require('../commandService'); const { createRecording } = require('./recording'); const { recoverRecordings, enforceRetention } = require('./storage'); +const { createTrainer } = require('./training'); +const { createDriving } = require('./driving'); +const { createGateway } = require('./socketGateway'); function createRuntime({ config, logger }) { const root = resolveDataPath('rover-learning'); const recordingsRoot = path.join(root, 'recordings'); + const recentActivity = []; + function activity(message, details = {}) { + recentActivity.push({ at: Date.now(), message, roverId: details.roverId || null, + detail: details.reason || details.error || details.model || null }); + if (recentActivity.length > 40) recentActivity.shift(); + } + const baseLogger = logger; + logger = Object.fromEntries(['info', 'warn', 'error'].map((level) => [level, (message, details) => { + baseLogger[level](message, details); activity(message, details); + }])); + const trainer = createTrainer({ root, config: config.training, logger }); + const driving = createDriving({ root, config, onChange: refreshEligibility, activity }); + let gateway; + let diskState = { bytes: null, free: null }; const sessions = new Map(); const unsubscribers = []; let running = false; @@ -30,6 +47,9 @@ function createRuntime({ config, logger }) { let eligibleRovers = new Set(); function hasHumanController(roverId) { + // Even manual inputs during autonomy remain excluded: model-influenced + // video must not silently become human demonstration training data. + if (driving.isActive(roverId)) return false; const record = roverManager.rovers.get(roverId); if (!record?.ws) return false; // Admin permission alone must not record the whole fleet. Require actual @@ -120,14 +140,16 @@ function createRuntime({ config, logger }) { } if (!running) return; const retention = await enforceRetention( - recordingsRoot, new Set([...sessions.values()].map((session) => session.id)), + recordingsRoot, new Set([...sessions.values()].map((session) => session.id).concat([...trainer.leases])), config.recording.maxGiB * 1024 ** 3, config.recording.minimumFreeGiB * 1024 ** 3, ); + diskState = retention; if (retention.paused) { if (!diskPaused) logger.warn('Recording paused for disk budget', retention); diskPaused = true; pending = []; pendingBytes = 0; + await trainer.cancel(); // Close active files to make them eligible for pruning on the next tick. await Promise.all([...sessions.keys()].map((id) => closeSession(id, 'disk-budget'))); return; @@ -156,6 +178,14 @@ function createRuntime({ config, logger }) { return false; }); } + if (running) { + try { + await trainer.tick(new Set([...sessions.values()].map((session) => session.id))); + } catch (error) { + // Training availability must not tear down otherwise healthy capture. + logger.error('Unable to schedule training', { error: error.message }); + } + } } function schedule() { @@ -175,7 +205,11 @@ function createRuntime({ config, logger }) { }); } - function killChildren() { for (const session of sessions.values()) session.kill(); } + function killChildren() { + driving.kill(); + trainer.kill(); + for (const session of sessions.values()) session.kill(); + } function onSignal() { // Other server services can exit the process before asynchronous cleanup // finishes. Signal children immediately, and keep an exit fallback too. @@ -186,7 +220,16 @@ function createRuntime({ config, logger }) { async function start() { await fs.mkdir(recordingsRoot, { recursive: true }); await recoverRecordings(recordingsRoot); + await trainer.start(); running = true; + driving.attach(); + gateway = createGateway({ root, driving, activity, getSystemState: () => ({ + recording: { paused: diskPaused, bytes: diskState.bytes, freeBytes: diskState.free, + pendingBytes, droppedPending, sessions: [...sessions.values()].map((session) => session.getStatus()) }, + training: trainer.getState(), + limits: { recording: config.recording, training: { ...config.training, python: undefined }, driving: config.driving }, + activity: [...recentActivity].reverse(), + }) }); eligibleRovers = currentEligibleRovers(); unsubscribers.push(subscribeCommandRecording((event) => append('command', event))); // ACK analytics also provide latency/type context; detailed acknowledgements @@ -226,17 +269,22 @@ function createRuntime({ config, logger }) { } if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000); wakeRequested = false; - logger.info('Rover learning recording enabled', { root }); + logger.info('Rover learning enabled', { root, training: config.training.enabled }); } async function stop() { if (stopPromise) return stopPromise; running = false; + gateway?.close(); clearTimeout(timer); for (const unsubscribe of unsubscribers.splice(0)) unsubscribe(); stopPromise = (async () => { + const trainingStop = trainer.stop(); + const drivingStop = driving.shutdown(); await maintenance.catch(() => undefined); const results = await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'disabled'))); + await trainingStop; + await drivingStop; process.off('SIGTERM', onSignal); process.off('SIGINT', onSignal); process.off('exit', killChildren); @@ -248,7 +296,7 @@ function createRuntime({ config, logger }) { return stopPromise; } - return { start, stop }; + return { start, stop, getTrainingState: trainer.getState }; } module.exports = { createRuntime }; diff --git a/server/src/services/roverLearningService/socketGateway.js b/server/src/services/roverLearningService/socketGateway.js new file mode 100644 index 00000000..393edb14 --- /dev/null +++ b/server/src/services/roverLearningService/socketGateway.js @@ -0,0 +1,111 @@ +// One complete card snapshot per subscriber; no telemetry-frequency UI events. +const io = require('../../globals/io'); +const roverManager = require('../roverManager'); +const { isVerified } = require('../verificationService'); +const { listModels } = require('./models'); +const { compatible } = require('./capabilities'); + +function createGateway({ root, driving, getSystemState, activity }) { + const bindings = new Map(); + const viewers = new Map(); + let models = []; + let modelError = null; + let lastCatalog = 0; + let closed = false; + let refreshing = false; + + function stateFor(socket, roverId) { + if (!isVerified(socket)) return { available: false }; + const visible = roverId && roverManager.canSeeRover(roverId, socket); + const id = visible ? roverId : null; + const session = id ? driving.getState(id) : null; + const canControl = Boolean(id && driving.canControl(socket, id)); + const system = getSystemState(); + return { + available: true, roverId: id, updatedAt: Date.now(), + control: { canStart: canControl && !session, canStop: Boolean(session && (canControl || session.socketId === socket.id)), + reason: !id ? 'Select a rover first' : !canControl ? 'You do not currently have control of this rover' : null }, + session, + models: models.filter((model) => model.trainedRovers?.some((rover) => roverManager.canSeeRover(rover, socket))) + .map((model) => ({ id: model.id, name: model.name, createdAt: model.createdAt, + trainingSteps: model.trainingSteps, examplesProcessed: model.examplesProcessed, + weightBytes: model.weightBytes, parameters: model.parameters, trainingLoss: model.trainingLoss, + evaluation: model.evaluation, controls: model.specification?.commands?.length || 0, + compatible: Boolean(id && compatible(model, roverManager.rovers.get(id))) })), + catalogError: modelError, + recording: { ...system.recording, sessions: system.recording.sessions.filter((s) => roverManager.canSeeRover(s.roverId, socket)) }, + training: { ...system.training, sessions: undefined }, + limits: system.limits, + activity: system.activity.filter((entry) => !entry.roverId || roverManager.canSeeRover(entry.roverId, socket)), + }; + } + + async function refresh() { + if (closed || refreshing) return; + refreshing = true; + try { + if (Date.now() - lastCatalog >= 15000) { + lastCatalog = Date.now(); + try { models = await listModels(root); modelError = null; } + catch (error) { modelError = error.message; } + } + if (closed) return; + for (const [socket, roverId] of viewers) socket.emit('roverLearning:state', stateFor(socket, roverId)); + } finally { refreshing = false; } + } + + function connect(socket) { + const subscribe = (payload = {}, reply) => { + const respond = typeof reply === 'function' ? reply : () => {}; + if (!isVerified(socket)) { respond({ error: 'Verification required' }); return; } + const roverId = typeof payload?.roverId === 'string' ? payload.roverId : null; + viewers.set(socket, roverId); + respond({ state: stateFor(socket, roverId) }); + refresh().catch((error) => activity('Card update failed', { error: error.message })); + }; + const action = async (payload = {}, reply) => { + const respond = typeof reply === 'function' ? reply : () => {}; + try { + if (!isVerified(socket)) throw new Error('Verification required'); + const roverId = String(payload.roverId || ''); + if (payload.action === 'start') await driving.start(socket, roverId, String(payload.modelId || '')); + else if (payload.action === 'stop') { + const session = driving.getState(roverId); + if (!session || !(driving.canControl(socket, roverId) || session.socketId === socket.id)) throw new Error('You cannot stop this controller'); + await driving.stop(roverId); + } else throw new Error('Unknown action'); + viewers.set(socket, roverId); + respond({ state: stateFor(socket, roverId) }); + } catch (error) { respond({ error: error.message }); } + refresh().catch((error) => activity('Card update failed', { error: error.message })); + }; + const unsubscribe = () => viewers.delete(socket); + const disconnect = () => { unsubscribe(); detach(socket); }; + const handlers = { 'roverLearning:subscribe': subscribe, 'roverLearning:action': action, + 'roverLearning:unsubscribe': unsubscribe, disconnect }; + for (const [name, handler] of Object.entries(handlers)) socket.on(name, handler); + bindings.set(socket, handlers); + } + + function detach(socket) { + for (const [name, handler] of Object.entries(bindings.get(socket) || {})) socket.off(name, handler); + bindings.delete(socket); + } + + io.on('connection', connect); + for (const socket of io.sockets.sockets.values()) connect(socket); + const timer = setInterval(() => { refresh().catch((error) => activity('Card update failed', { error: error.message })); }, 1000); + refresh().catch((error) => activity('Catalog unavailable', { error: error.message })); + return { + close() { + closed = true; + clearInterval(timer); + io.off('connection', connect); + for (const socket of viewers.keys()) socket.emit('roverLearning:state', { available: false }); + viewers.clear(); + for (const socket of bindings.keys()) detach(socket); + }, + }; +} + +module.exports = { createGateway }; diff --git a/server/src/services/roverLearningService/storage.js b/server/src/services/roverLearningService/storage.js index e174b628..7fd306db 100644 --- a/server/src/services/roverLearningService/storage.js +++ b/server/src/services/roverLearningService/storage.js @@ -56,4 +56,4 @@ async function enforceRetention(root, activeIds, maxBytes, minimumFreeBytes) { return { bytes, free, paused: bytes >= maxBytes || free < minimumFreeBytes }; } -module.exports = { recoverRecordings, enforceRetention }; +module.exports = { recoverRecordings, enforceRetention, listRecordings }; diff --git a/server/src/services/roverLearningService/training.js b/server/src/services/roverLearningService/training.js new file mode 100644 index 00000000..7280518d --- /dev/null +++ b/server/src/services/roverLearningService/training.js @@ -0,0 +1,177 @@ +// One owned finite worker at a time. Leases are acquired in the runtime's +// serialized maintenance pass, before retention can next inspect recordings. +const fs = require('node:fs/promises'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { randomInt } = require('node:crypto'); +const { listRecordings } = require('./storage'); + +function createTrainer({ root, config, logger }) { + const leases = new Set(); + const trainingRoot = path.join(root, 'training'); + let ledger = {}; + let child = null; + let completion = Promise.resolve(); + let stopped = false; + let nextRun = 0; + let startupError = false; + let state = { status: config.enabled ? 'waiting' : 'disabled', steps: 0, examples: 0 }; + + async function start() { + if (!config.enabled) return; + try { + await fs.mkdir(trainingRoot, { recursive: true }); + await fs.mkdir(path.join(root, 'models'), { recursive: true }); + // Only unpublished staging directories from an interrupted old worker + // are disposable. Permanent UUID model directories are never removed. + for (const entry of await fs.readdir(path.join(root, 'models'), { withFileTypes: true })) { + if (entry.isDirectory() && /^\.[0-9a-f-]{36}\.pending$/.test(entry.name)) { + await fs.rm(path.join(root, 'models', entry.name), { recursive: true }); + } + } + try { + const saved = JSON.parse(await fs.readFile(path.join(trainingRoot, 'schedule.json'), 'utf8')); + ledger = saved.sessions; + state = { ...saved.progress, status: 'waiting' }; + if (!ledger || typeof ledger !== 'object') throw new Error('Invalid training schedule'); + } catch (error) { if (error.code !== 'ENOENT') throw error; } + } catch (error) { + startupError = true; + state = { ...state, status: 'error', error: error.message }; + logger.error('Training initialization failed; recording remains available', { error: error.message }); + } + } + + async function saveLedger() { + const filename = path.join(trainingRoot, 'schedule.json'); + await fs.writeFile(`${filename}.tmp`, JSON.stringify({ sessions: ledger, progress: state })); + await fs.rename(`${filename}.tmp`, filename); + } + + async function tick(activeIds) { + if (!config.enabled || startupError || stopped || child || Date.now() < nextRun) return; + nextRun = Date.now() + config.intervalSeconds * 1000; + const recordings = await listRecordings(path.join(root, 'recordings')); + const present = new Set(recordings.map((recording) => recording.id)); + // Scheduling history is bounded by recording retention, unlike models. + for (const id of Object.keys(ledger)) if (!present.has(id)) delete ledger[id]; + const eligible = []; + for (const recording of recordings) { + if (activeIds.has(recording.id) || ledger[recording.id]?.rejected + || (ledger[recording.id]?.rounds || 0) >= config.roundsPerSession) continue; + try { + const metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8')); + if (!metadata.endedAt || metadata.error || metadata.droppedEvents || metadata.reason === 'interrupted') continue; + eligible.push(recording.id); + } catch (error) { + ledger[recording.id] = { rejected: `Unreadable session metadata: ${error.message}` }; + } + } + // Random cohorts mix old and new retained sessions without repeatedly + // favoring the first files in directory order. + for (let index = eligible.length - 1; index > 0; index -= 1) { + const other = randomInt(index + 1); + [eligible[index], eligible[other]] = [eligible[other], eligible[index]]; + } + const selected = eligible.slice(0, 4); + nextRun = Date.now() + config.intervalSeconds * 1000; + if (!selected.length) { + state = { ...state, status: 'waiting' }; + await saveLedger(); + return; + } + const jobPath = path.join(trainingRoot, 'job.json'); + await fs.writeFile(jobPath, JSON.stringify({ root, config, sessions: selected })); + if (stopped) return; + selected.forEach((id) => leases.add(id)); + state = { ...state, status: 'training', sessions: selected }; + state.startedAt = Date.now(); + child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], { + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1', OMP_NUM_THREADS: String(config.threads), MKL_NUM_THREADS: String(config.threads) }, + }); + let output = ''; + let stderr = ''; + let result = null; + let errorMessage = null; + let forcedTimer; + const worker = child; + const deadline = setTimeout(() => { + worker.kill('SIGTERM'); + forcedTimer = setTimeout(() => worker.kill('SIGKILL'), 5000); + }, (config.maxJobSeconds + 15) * 1000); + child.on('error', (error) => { errorMessage = error.message; }); + child.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-8192); }); + child.stdout.on('data', (chunk) => { + output += chunk; + if (output.length > 1024 * 1024) { + errorMessage = 'Training worker output exceeded protocol bound'; + worker.kill('SIGKILL'); + return; + } + let newline; + while ((newline = output.indexOf('\n')) >= 0) { + const line = output.slice(0, newline); + output = output.slice(newline + 1); + try { + const message = JSON.parse(line); + if (message.kind === 'complete') result = message; + if (message.kind === 'error') errorMessage = message.message; + if (message.kind === 'skipped') logger.warn('Training skipped recording', message); + if (message.kind === 'dataset') state = { ...state, lastSampledWindows: message.sampled, lastAvailableWindows: message.windows }; + if (message.kind === 'progress') state = { ...state, steps: message.steps, examples: message.examples, loss: message.loss }; + } catch { errorMessage = 'Invalid training worker message'; } + } + }); + completion = new Promise((resolve) => { + child.once('close', async (code, signal) => { + clearTimeout(deadline); + clearTimeout(forcedTimer); + try { + if (code !== 0 || errorMessage || !result) { + state = { ...state, status: stopped ? 'stopped' : 'error', error: errorMessage || `Worker exited ${code ?? signal}` }; + if (!stopped) logger.error('Training job failed', { error: state.error, stderr }); + } else { + for (const [id, reason] of Object.entries(result.rejected || {})) ledger[id] = { rejected: reason }; + if (result.trained) { + for (const id of result.accepted) ledger[id] = { rounds: (ledger[id]?.rounds || 0) + 1 }; + } + state = { ...state, status: stopped ? 'stopped' : 'waiting', steps: result.steps ?? state.steps, + completedAt: Date.now(), elapsedSeconds: result.elapsedSeconds ?? null, error: null, + examples: result.examples ?? state.examples, loss: result.loss ?? null, reason: result.reason ?? null }; + await saveLedger(); + logger.info('Training job completed', { ...state, model: result.published?.name || null }); + } + } catch (error) { + state = { ...state, status: 'error', error: error.message }; + logger.error('Training result persistence failed', { error: error.message }); + } finally { + leases.clear(); + child = null; + nextRun = Date.now() + config.intervalSeconds * 1000; + resolve(); + } + }); + }); + } + + async function cancel() { + if (!child) return; + const worker = child; + worker.kill('SIGTERM'); + const timer = setTimeout(() => worker.kill('SIGKILL'), 5000); + try { await completion; } finally { clearTimeout(timer); } + } + + return { + leases, start, tick, cancel, + getState: () => ({ ...state, nextAttemptAt: nextRun || null, leasedSessions: leases.size, + visitedSessions: Object.values(ledger).filter((entry) => entry.rounds).length, + skippedSessions: Object.values(ledger).filter((entry) => entry.rejected).length, + rejectionReasons: [...new Set(Object.values(ledger).filter((entry) => entry.rejected).map((entry) => entry.rejected))].slice(0, 10) }), + stop: async () => { stopped = true; await cancel(); }, + kill: () => child?.kill('SIGKILL'), + }; +} + +module.exports = { createTrainer }; diff --git a/server/src/services/roverLearningService/workers/dataset.py b/server/src/services/roverLearningService/workers/dataset.py new file mode 100644 index 00000000..40cfeb5e --- /dev/null +++ b/server/src/services/roverLearningService/workers/dataset.py @@ -0,0 +1,173 @@ +"""Read closed recordings with causal state/action alignment and bounded memory.""" +import bisect +import collections +import copy +import json +import math +from pathlib import Path + +import av +import numpy as np + +from policy import SLOTS, VALUES, HEIGHT, WIDTH, encode_command, sensor_vector + + +def load_events(directory): + metadata = json.loads((directory / 'session.json').read_text()) + if (metadata.get('version') != 1 or not metadata.get('endedAt') or metadata.get('error') + or metadata.get('droppedEvents') or metadata.get('reason') == 'interrupted'): + raise ValueError('Incomplete/interrupted recording or lost events') + if metadata.get('video', {}).get('origin') != 'unix': + raise ValueError('Unsupported video time origin') + video_result = metadata.get('video', {}).get('result') or {} + if video_result.get('signal') == 'SIGKILL' or video_result.get('code') not in (0, 255, None): + raise ValueError('Video recorder failed or was forcibly killed') + event_file = directory / 'events.ndjson' + if event_file.stat().st_size > 64 * 1024 * 1024: + raise ValueError('Event file exceeds 64 MiB read bound') + events = [] + with event_file.open() as stream: + for line in stream: + event = json.loads(line) + if event['kind'] == 'captureGap': + raise ValueError('Recording contains a capture gap') + if not isinstance(event.get('ts'), (int, float)) or not math.isfinite(event['ts']): + raise ValueError('Invalid event timestamp') + events.append(event) + if not events or not any(e['kind'] == 'sessionEnd' for e in events): + raise ValueError('Missing completed event stream') + # File order can differ across rotation buffering. Monotonic time detects + # wall-clock steps; sorting Unix timestamps alone would hide those steps. + timed = sorted(events, key=lambda e: int(e['monotonicNs'])) + origin = timed[0] + for event in timed: + drift = (event['ts'] - origin['ts']) - (int(event['monotonicNs']) - int(origin['monotonicNs'])) / 1e6 + if abs(drift) > 250: + raise ValueError('Server wall clock shifted during recording') + return metadata, sorted(events, key=lambda e: e['ts']) + + +def prepare(directory, schema): + metadata, events = load_events(directory) + requests = {} + accepted = set() + failed = set() + sensors, dispatches, boundaries = [], [], [] + for event in events: + data = event['data'] + if event['kind'] == 'sensor': + sensors.append((event['ts'], sensor_vector(data.get('sensors') or {}))) + if event['kind'] in ('driver', 'turn:activeDriver', 'lock', 'private'): + boundaries.append(event['ts']) + if event['kind'] != 'command': + continue + if data.get('phase') == 'requested' and data.get('source') == 'client': + requests[data.get('requestId')] = data + if data.get('phase') == 'result' and data.get('result', {}).get('id'): + accepted.add(data['result']['id']) + if data.get('phase') == 'acknowledged' and data.get('acknowledgement', {}).get('error'): + failed.add(data.get('commandId')) + if data.get('phase') == 'dispatched': + dispatches.append(data) + if not sensors: + raise ValueError('No sensor observations') + candidate = copy.deepcopy(schema) + history, targets = [], [] + for data in dispatches: + encoded = encode_command(data.get('command') or {}, candidate, extend=True) + if encoded is None: + continue + stamp = data['ts'] + history.append((stamp, *encoded)) + request = requests.get(data.get('requestId')) + if (data.get('source') == 'client' and request + and data.get('commandId') in accepted and data.get('commandId') not in failed): + target_time = request['ts'] + if not 0 <= stamp - target_time <= 1000: + raise ValueError('Invalid request/dispatch timing') + targets.append((target_time, *encoded)) + else: + # Server safety overrides belong in history, never demonstration + # targets. Exclude windows around them rather than teaching inaction. + boundaries.append(stamp) + if not targets: + raise ValueError('No accepted client operating commands') + schema[:] = candidate + return metadata, sensors, sorted(history, key=lambda x: x[0]), sorted(targets, key=lambda x: x[0]), sorted(boundaries) + + +def windows(directory, schema, cancelled=lambda: False): + directory = Path(directory) + metadata, sensors, history, targets, boundaries = prepare(directory, schema) + sensor_times = [row[0] for row in sensors] + target_times = [row[0] for row in targets] + available = np.zeros(SLOTS, dtype=np.float32) + for _, slot, _ in history: + available[slot] = 1 + command_values = np.zeros((SLOTS, VALUES), dtype=np.float32) + command_times = np.full(SLOTS, -np.inf) + cursor = 0 + recent = collections.deque(maxlen=32) + last_sample, last_pts = -math.inf, -math.inf + with av.open(str(directory / 'video.mkv')) as container: + stream = container.streams.video[0] + stream.codec_context.thread_count = 1 + for frame in container.decode(stream): + if cancelled(): + return + if frame.pts is None: + raise ValueError('Video frame has no timestamp') + stamp = float(frame.pts * frame.time_base) * 1000 + if not math.isfinite(stamp) or stamp < last_pts: + raise ValueError('Invalid/nonmonotonic video timestamps') + last_pts = stamp + if not metadata['startedAt'] - 2000 <= stamp <= metadata['endedAt'] + 2000: + raise ValueError('Video PTS is not aligned to session Unix time') + if stamp - last_sample < 99: + continue + last_sample = stamp + while cursor < len(history) and history[cursor][0] < stamp: + when, slot, values = history[cursor] + command_values[slot] = values + command_times[slot] = when + cursor += 1 + sensor_index = bisect.bisect_right(sensor_times, stamp) - 1 + if sensor_index < 0 or stamp - sensor_times[sensor_index] > 1000: + recent.clear() + continue + seen = np.isfinite(command_times).astype(np.float32) + ages = np.minimum((stamp - command_times) / 10000, 1).astype(np.float32) + state = np.concatenate((sensors[sensor_index][1], command_values.flatten(), ages, seen)) + pixels = frame.to_ndarray(width=WIDTH, height=HEIGHT, format='rgb24').transpose(2, 0, 1).copy() + recent.append((stamp, pixels, state)) + chosen = [] + for offset in (900, 600, 300, 0): + options = [row for row in recent if row[0] <= stamp - offset + 0.01] + if not options or stamp - offset - options[-1][0] > 200: + break + chosen.append(options[-1]) + if len(chosen) != 4 or stamp + 100 > metadata['endedAt']: + continue + if bisect.bisect_right(boundaries, stamp + 100) > bisect.bisect_left(boundaries, chosen[0][0]): + continue + start = bisect.bisect_left(target_times, stamp) + end = bisect.bisect_left(target_times, stamp + 100) + actions = np.zeros(SLOTS, dtype=np.float32) + values = np.zeros((SLOTS, VALUES), dtype=np.float32) + ambiguous = False + families = set() + for _, slot, vector in targets[start:end]: + template = schema[slot]['template'] + family = template['type'] + if family == 'peripheral': + peripheral = template.get('peripheral') or {} + family += ':' + str(peripheral.get('id')) + ':' + str(peripheral.get('control')) + if ((actions[slot] and (not schema[slot]['fields'] or not np.array_equal(values[slot], vector))) + or (not actions[slot] and family in families)): + ambiguous = True + break + families.add(family) + actions[slot], values[slot] = 1, vector + if ambiguous: + continue + yield np.stack([row[1] for row in chosen]), np.stack([row[2] for row in chosen]), actions, values, available diff --git a/server/src/services/roverLearningService/workers/infer.py b/server/src/services/roverLearningService/workers/infer.py new file mode 100644 index 00000000..9013108d --- /dev/null +++ b/server/src/services/roverLearningService/workers/infer.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Live policy worker. JSON state on stdin, proposals on stdout; no rover writes.""" +import argparse +import collections +import hashlib +import json +import math +from pathlib import Path +import sys +import threading +import time + +import av +import numpy as np +import torch + +from policy import Policy, VERSION, SLOTS, VALUES, HEIGHT, WIDTH, encode_command, sensor_vector + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model', required=True) + parser.add_argument('--url', required=True) + parser.add_argument('--threads', type=int, default=4) + parser.add_argument('--threshold', type=float, default=0.7) + args = parser.parse_args() + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + directory = Path(args.model) + metadata = json.loads((directory / 'model.json').read_text()) + if metadata['specification']['version'] != VERSION: + raise ValueError('Model specification is incompatible') + if hashlib.sha256((directory / 'weights.pt').read_bytes()).hexdigest() != metadata['sha256']: + raise ValueError('Model weights checksum mismatch') + schema = metadata['specification']['commands'] + model = Policy().eval() + model.load_state_dict(torch.load(directory / 'weights.pt', map_location='cpu', weights_only=True)) + observations = collections.deque(maxlen=2048) + mutex = threading.Lock() + closed = threading.Event() + + def read_input(): + try: + for line in sys.stdin: + event = json.loads(line) + with mutex: + observations.append(event) + finally: + closed.set() + + threading.Thread(target=read_input, daemon=True).start() + values = np.zeros((SLOTS, VALUES), dtype=np.float32) + last_commands = np.full(SLOTS, -np.inf) + sensors, sensor_time = {}, 0 + history = collections.deque(maxlen=32) + latched = set() + sampled = 0 + with av.open(args.url, options={'rtsp_transport': 'tcp', 'timeout': '3000000', + 'use_wallclock_as_timestamps': '1'}) as video: + stream = video.streams.video[0] + stream.codec_context.thread_count = 1 + for frame in video.decode(stream): + if closed.is_set(): + return + if frame.pts is None: + continue + stamp = float(frame.pts * frame.time_base) * 1000 + now = time.time() * 1000 + if not math.isfinite(stamp) or abs(now - stamp) > 2000: + raise ValueError('Video time is stale or not aligned to server time') + if stamp - sampled < 99: + continue + sampled = stamp + with mutex: + while observations and observations[0]['ts'] < stamp: + event = observations.popleft() + if event['kind'] == 'sensor': + sensors, sensor_time = event['data'], event['ts'] + elif event['kind'] == 'command': + try: + encoded = encode_command(event['data'], schema) + if encoded: + slot, vector = encoded + values[slot], last_commands[slot] = vector, event['ts'] + except ValueError: + pass # Unseen manual controls do not stop autonomy. + if stamp - sensor_time > 1000: + history.clear() + continue + state = np.concatenate((sensor_vector(sensors), values.flatten(), + np.minimum((stamp - last_commands) / 10000, 1).astype(np.float32), + np.isfinite(last_commands).astype(np.float32))) + pixels = frame.to_ndarray(width=WIDTH, height=HEIGHT, format='rgb24').transpose(2, 0, 1).copy() + history.append((stamp, pixels, state)) + selected = [] + for offset in (900, 600, 300, 0): + candidates = [entry for entry in history if entry[0] <= stamp - offset + .01] + if not candidates or stamp - offset - candidates[-1][0] > 200: + break + selected.append(candidates[-1]) + if len(selected) != 4: + continue + before = time.perf_counter() + with torch.inference_mode(): + logits, numeric = model(torch.from_numpy(np.stack([row[1] for row in selected])).unsqueeze(0).float() / 255, + torch.from_numpy(np.stack([row[2] for row in selected])).unsqueeze(0)) + probabilities = logits[0].sigmoid().tolist() + predictions = numeric[0].tolist() + if not all(math.isfinite(value) for value in probabilities): + raise ValueError('Nonfinite model output') + families = {} + active = set() + for slot, descriptor in enumerate(schema): + if probabilities[slot] < args.threshold: + continue + active.add(slot) + command = json.loads(json.dumps(descriptor['template'])) + family = command['type'] + if family == 'peripheral': + family += ':' + str(command['peripheral']['id']) + ':' + str(command['peripheral']['control']) + for index, field in enumerate(descriptor['fields']): + number = predictions[slot][index] + if not math.isfinite(number): + raise ValueError('Nonfinite control value') + number = math.tan(max(-.98, min(.98, number)) * math.pi / 2) * field['scale'] + target = command + for key in field['path'][:-1]: + target = target[key] + target[field['path'][-1]] = round(number) if field['integer'] else number + discrete = command['type'] in ('headlight', 'laser', 'horn', 'song', 'raw') + if family not in families or probabilities[slot] > families[family][0]: + families[family] = (probabilities[slot], command, slot, discrete) + latched.intersection_update(active) + # Choose the winning action before latching, so a held toggle cannot + # expose a competing lower-confidence toggle on the following tick. + commands = [command for _, command, slot, discrete in families.values() + if not discrete or slot not in latched] + for _, _, slot, discrete in families.values(): + if discrete: + latched.add(slot) + print(json.dumps({'kind': 'prediction', 'frameAt': stamp, 'sensorAt': sensor_time, + 'latencyMs': (time.perf_counter() - before) * 1000, + 'commands': commands}, allow_nan=False), flush=True) + + +if __name__ == '__main__': + try: + main() + except Exception as error: + print(json.dumps({'kind': 'error', 'message': str(error)}), flush=True) + raise SystemExit(1) diff --git a/server/src/services/roverLearningService/workers/policy.py b/server/src/services/roverLearningService/workers/policy.py new file mode 100644 index 00000000..79bc97e6 --- /dev/null +++ b/server/src/services/roverLearningService/workers/policy.py @@ -0,0 +1,131 @@ +"""Versioned, fixed-size vision/control policy and command encoding. + +Slots describe command shapes, not arbitrary executable instructions. Numeric +leaves are regressed; string/bool/list values remain exact template constants. +No command is sent by this module. Execution must use server authorization. +""" +import copy +import json +import math + +import numpy as np +import torch +from torch import nn + +VERSION = 1 +SLOTS = 64 +VALUES = 8 +FRAMES = 4 +HEIGHT, WIDTH = 120, 160 +OPERATING_TYPES = {"drive", "motors", "servo", "headlight", "laser", "horn", "peripheral", "song", "raw"} +SENSORS = [ + ("bumpsAndWheelDrops.bumpLeft", 1), ("bumpsAndWheelDrops.bumpRight", 1), + ("bumpsAndWheelDrops.wheelDropLeft", 1), ("bumpsAndWheelDrops.wheelDropRight", 1), + ("cliffLeft", 1), ("cliffFrontLeft", 1), ("cliffFrontRight", 1), ("cliffRight", 1), + ("wall", 1), ("virtualWall", 1), ("wallSignal", 4096), + ("voltageMv", 16000), ("currentMa", 3000), ("batteryChargeMah", 4000), + ("batteryCapacityMah", 4000), ("batteryTemperatureC", 50), + ("chargingState.code", 5), ("chargingSources.homeBase", 1), ("oiMode.code", 4), + ("wheelSpeedsMmPerSecond.left", 500), ("wheelSpeedsMmPerSecond.right", 500), + ("wheelLeftCurrentMa", 2000), ("wheelRightCurrentMa", 2000), + ("mainBrushCurrentMa", 2000), ("sideBrushCurrentMa", 2000), + ("lightBumpLeftSignal", 4096), ("lightBumpRightSignal", 4096), + ("cliffLeftSignal", 4096), ("cliffRightSignal", 4096), + ("distanceMm", 500), ("angleDeg", 180), ("stasis", 1), +] +STATE_SIZE = len(SENSORS) * 2 + SLOTS * (VALUES + 2) + + +def normalize(value, scale): + return math.atan(float(value) / scale) * 2 / math.pi + + +def sensor_vector(sensors): + values, present = [], [] + for name, scale in SENSORS: + value = sensors + for key in name.split('.'): + value = value.get(key) if isinstance(value, dict) else None + known = isinstance(value, (int, float)) and math.isfinite(value) + values.append(normalize(value, scale) if known else 0) + present.append(float(known)) + return np.array(values + present, dtype=np.float32) + + +def describe_command(command): + if command.get('type') not in OPERATING_TYPES: + return None + if len(json.dumps(command)) > 4096: + raise ValueError('Operating command exceeds 4096-byte schema limit') + values, fields = [], [] + + def visit(value, path=()): + if isinstance(value, dict): + return {key: visit(item, (*path, key)) for key, item in sorted(value.items())} + if isinstance(value, (int, float)) and not isinstance(value, bool): + if not math.isfinite(value) or len(values) >= VALUES: + raise ValueError('Nonfinite command or too many numeric control fields') + scale = {'drive': 500, 'motors': 255, 'servo': 180, 'peripheral': 255, + 'horn': 1000, 'song': 127}.get(command['type'], 1) + fields.append({'path': list(path), 'scale': scale, 'integer': isinstance(value, int)}) + values.append(normalize(value, scale)) + return {'$number': len(values) - 1} + # Arrays (e.g. songs) are discrete exact demonstrations, not variable + # length regression outputs. Never reinterpret raw OI bytes as numbers. + return copy.deepcopy(value) + + template = visit(command) + return {'template': template, 'fields': fields}, values + + +def encode_command(command, schema, extend=False): + described = describe_command(command) + if described is None: + return None + descriptor, values = described + try: + slot = schema.index(descriptor) + except ValueError: + if not extend: + raise ValueError('Command shape missing from model schema') + if len(schema) >= SLOTS: + raise ValueError('64 command shapes exhausted; recording skipped, not silently truncated') + slot = len(schema) + schema.append(descriptor) + vector = np.zeros(VALUES, dtype=np.float32) + vector[:len(values)] = values + return slot, vector + + +class Policy(nn.Module): + def __init__(self): + super().__init__() + self.vision = nn.Sequential( + nn.Conv2d(3, 24, 5, 2, 2), nn.ReLU(), + nn.Conv2d(24, 48, 3, 2, 1), nn.ReLU(), + nn.Conv2d(48, 96, 3, 2, 1), nn.ReLU(), + nn.Conv2d(96, 128, 3, 2, 1), nn.ReLU(), + nn.AdaptiveAvgPool2d((3, 4)), nn.Flatten(), + nn.Linear(128 * 3 * 4, 128), nn.ReLU(), + ) + self.history = nn.GRU(128 + STATE_SIZE, 128, batch_first=True) + self.actions = nn.Linear(128, SLOTS) + self.values = nn.Linear(128, SLOTS * VALUES) + + def forward(self, images, state): + batch, frames = images.shape[:2] + encoded = self.vision(images.reshape(-1, 3, HEIGHT, WIDTH)).reshape(batch, frames, -1) + history, _ = self.history(torch.cat((encoded, state), dim=-1)) + last = history[:, -1] + return self.actions(last), self.values(last).reshape(batch, SLOTS, VALUES).tanh() + + +def specification(schema): + return {'version': VERSION, 'architecture': 'spatial-cnn-gru-v1', + 'frames': FRAMES, 'height': HEIGHT, 'width': WIDTH, + 'historyStrideMs': 300, 'decisionIntervalMs': 100, + 'stateSize': STATE_SIZE, 'sensorFields': SENSORS, + 'normalization': '2/pi * atan(value/scale); missing sensors have mask=0', + 'maxCommandShapes': SLOTS, 'numericFieldsPerShape': VALUES, + 'commands': schema, 'actionSemantics': 'Events in the next 100 ms; zero means no new command', + 'timing': 'Server-receipt alignment; browser display latency unmeasured'} diff --git a/server/src/services/roverLearningService/workers/requirements.txt b/server/src/services/roverLearningService/workers/requirements.txt new file mode 100644 index 00000000..9ea3ff91 --- /dev/null +++ b/server/src/services/roverLearningService/workers/requirements.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +torch==2.14.0+cpu +numpy==2.5.3 +av==18.1.0 diff --git a/server/src/services/roverLearningService/workers/train.py b/server/src/services/roverLearningService/workers/train.py new file mode 100644 index 00000000..70cc1b42 --- /dev/null +++ b/server/src/services/roverLearningService/workers/train.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""One finite CPU training job; Node owns scheduling, leases, and shutdown. + +Usage: python train.py --job /path/to/job.json +Dependencies are packaged at image build time, never installed at runtime. +""" +import argparse +import copy +import hashlib +import json +import os +from pathlib import Path +import random +import signal +import time +import uuid + + +def emit(kind, **data): + print(json.dumps({'kind': kind, **data}, allow_nan=False), flush=True) + + +def atomic_torch_save(torch, value, path): + temporary = path.with_suffix('.tmp') + with temporary.open('wb') as stream: + torch.save(value, stream) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--job') + parser.add_argument('--check-dependencies', action='store_true', help='Describe the installed production policy and exit without creating storage') + args = parser.parse_args() + # --help works even before Python dependencies are installed. + import numpy as np + import av + import torch + from torch.nn import functional as functional + from dataset import windows + from policy import Policy, VERSION, SLOTS, VALUES, specification + + if args.check_dependencies: + emit('dependencies', torch=str(torch.__version__), numpy=str(np.__version__), av=str(av.__version__), + parameters=sum(p.numel() for p in Policy().parameters()), specification=specification([])) + return + if not args.job: + parser.error('--job is required unless --check-dependencies is used') + + job = json.loads(Path(args.job).read_text()) + root = Path(job['root']) + config = job['config'] + torch.set_num_threads(config['threads']) + torch.set_num_interop_threads(1) + # Training yields CPU priority to the Node/media processes. Thread count is + # an additional bound, not a claim to reserve specific physical cores. + os.nice(10) + stopped = False + + def stop(_signal, _frame): + nonlocal stopped + stopped = True + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + started = time.monotonic() + cancelled = lambda: stopped or time.monotonic() - started >= config['maxJobSeconds'] + model = Policy().cpu() + optimizer = torch.optim.AdamW(model.parameters(), lr=0.0001) + resume_path = root / 'training' / 'resume.pt' + schema, steps, examples, last_published = [], 0, 0, 0 + trained_rovers = set() + control_profiles = [] + if resume_path.exists(): + saved = torch.load(resume_path, map_location='cpu', weights_only=True) + if saved['version'] != VERSION: + raise ValueError('Resume schema incompatible; refusing to reinterpret existing weights') + model.load_state_dict(saved['weights']) + optimizer.load_state_dict(saved['optimizer']) + schema = saved['schema'] + steps, examples = saved['steps'], saved['examples'] + last_published = saved['lastPublished'] + trained_rovers = set(saved['trainedRovers']) + control_profiles = saved.get('controlProfiles', []) + initial_steps = steps + pool, accepted, rejected = [], [], {} + sessions = list(job['sessions']) + random.shuffle(sessions) + per_session = max(1, config['maxSamples'] // max(1, len(sessions))) + for session_id in sessions: + if cancelled(): + break + candidate_schema = copy.deepcopy(schema) + samples, seen = [], 0 + try: + directory = root / 'recordings' / session_id + for sample in windows(directory, candidate_schema, cancelled): + seen += 1 + # Uniform reservoir across the whole recording, not just the + # opening seconds. Frames remain uint8 until a batch is used. + if len(samples) < per_session: + samples.append(sample) + else: + index = random.randrange(seen) + if index < per_session: + samples[index] = sample + if cancelled(): + break # Do not call a partially decoded recording validated. + if not samples: + raise ValueError('No complete, aligned observation/action windows') + schema = candidate_schema + pool.extend(samples) + accepted.append(session_id) + metadata = json.loads((directory / 'session.json').read_text()) + trained_rovers.add(str(metadata['roverId'])) + profile = metadata.get('controlProfile') + if profile is not None and profile not in control_profiles: + control_profiles.append(profile) + emit('dataset', sessionId=session_id, windows=seen, sampled=len(samples)) + except (ValueError, KeyError, TypeError, OSError, EOFError, IndexError, OverflowError, av.error.FFmpegError) as error: + rejected[session_id] = str(error)[:500] + emit('skipped', sessionId=session_id, reason=rejected[session_id]) + if len(pool) < config['minimumSamples'] or not any(sample[2].any() for sample in pool) or cancelled(): + emit('complete', trained=False, accepted=[], rejected=rejected, samples=len(pool), + reason='Stopped/time limit' if cancelled() else 'Waiting for more usable samples') + return + + numeric_mask = torch.zeros(SLOTS, VALUES) + for slot, descriptor in enumerate(schema): + numeric_mask[slot, :len(descriptor['fields'])] = 1 + # Balance sparse events without assuming that every time step should fire + # every control. This is training loss, not a driving-quality evaluation. + positives = np.sum([sample[2] for sample in pool], axis=0) + opportunities = np.sum([sample[4] for sample in pool], axis=0) + positive_weight = torch.from_numpy(np.clip((opportunities - positives) / np.maximum(positives, 1), 1, 20)).float() + losses = [] + model.train() + for _ in range(config['passesPerJob']): + random.shuffle(pool) + for offset in range(0, len(pool), config['batchSize']): + if cancelled() or steps - initial_steps >= config['maxStepsPerJob']: + break + batch = pool[offset:offset + config['batchSize']] + images, states, targets, values, available = [torch.from_numpy(np.stack(items)) for items in zip(*batch)] + images = images.float().div_(255) + optimizer.zero_grad(set_to_none=True) + action_logits, predictions = model(images, states) + event_loss = functional.binary_cross_entropy_with_logits(action_logits, targets, + pos_weight=positive_weight, reduction='none') + event_loss = (event_loss * available).sum() / available.sum().clamp_min(1) + mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1) + value_loss = ((predictions - values).square() * mask).sum() / mask.sum().clamp_min(1) + loss = event_loss + value_loss + if not torch.isfinite(loss): + raise ValueError('Nonfinite training loss; checkpoint not saved') + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1) + optimizer.step() + steps += 1 + examples += len(batch) + losses.append(loss.item()) + if steps % 10 == 0: + emit('progress', steps=steps, examples=examples, loss=loss.item()) + if cancelled() or steps - initial_steps >= config['maxStepsPerJob']: + break + if steps == initial_steps: + emit('complete', trained=False, accepted=[], rejected=rejected, reason='No training steps completed') + return + + published = None + if steps - last_published >= config['checkpointEverySteps']: + adjectives = ['amber', 'curious', 'gentle', 'quiet', 'silver', 'bright', 'merry', 'sleepy'] + animals = ['otter', 'finch', 'badger', 'robin', 'fox', 'heron', 'marten', 'wren'] + model_id = str(uuid.uuid4()) + name = f'{random.choice(adjectives)}-{random.choice(animals)}-{model_id[:8]}' + staging = root / 'models' / f'.{model_id}.pending' + staging.mkdir() + weights_path = staging / 'weights.pt' + atomic_torch_save(torch, model.state_dict(), weights_path) + published = { + 'id': model_id, 'name': name, 'createdAt': int(time.time() * 1000), + 'trainingSteps': steps, 'examplesProcessed': examples, + 'parameters': sum(p.numel() for p in model.parameters()), + 'weightBytes': weights_path.stat().st_size, + 'sha256': hashlib.sha256(weights_path.read_bytes()).hexdigest(), + 'specification': specification(schema), 'trainedRovers': sorted(trained_rovers), + 'controlProfiles': control_profiles, + 'sourceSessions': accepted, 'trainingLoss': sum(losses) / len(losses), + 'evaluation': 'unevaluated', 'dtype': 'float32', 'torchVersion': str(torch.__version__), + } + (staging / 'model.json').write_text(json.dumps(published, indent=2, allow_nan=False)) + # A catalog reader sees both weights and metadata, or neither. Existing + # versions are never overwritten or deleted by the trainer. + staging.rename(root / 'models' / model_id) + last_published = steps + atomic_torch_save(torch, { + 'version': VERSION, 'weights': model.state_dict(), 'optimizer': optimizer.state_dict(), + 'schema': schema, 'steps': steps, 'examples': examples, + 'lastPublished': last_published, 'trainedRovers': sorted(trained_rovers), + 'controlProfiles': control_profiles, + }, resume_path) + emit('complete', trained=True, accepted=accepted, rejected=rejected, + steps=steps, examples=examples, loss=sum(losses) / len(losses), + published=published, elapsedSeconds=time.monotonic() - started) + + +if __name__ == '__main__': + try: + main() + except Exception as error: + emit('error', message=str(error)) + raise SystemExit(1) diff --git a/webui/src/components/VipPanel/index.jsx b/webui/src/components/VipPanel/index.jsx index 91ec758f..ea3ff01e 100644 --- a/webui/src/components/VipPanel/index.jsx +++ b/webui/src/components/VipPanel/index.jsx @@ -11,6 +11,7 @@ import VipVerificationCard from '../vip/VipVerificationCard.jsx'; import VipIdentityCard from '../vip/VipIdentityCard.jsx'; import VipPrivateRoverAccessCard from '../vip/VipPrivateRoverAccessCard.jsx'; import VipProfileImageCard from '../vip/VipProfileImageCard.jsx'; +import VipRoverLearningCard from '../vip/VipRoverLearningCard.jsx'; export default function VipPanel() { const session = useSessionSelector((state) => state.session); @@ -54,6 +55,9 @@ export default function VipPanel() { return (
+ {isVerified && session?.features?.roverLearning ? ( +
+ ) : null} {!isVerified ? (
value == null ? 'Unavailable' : String(value); +const bytes = (value) => value == null ? 'Unavailable' : `${(value / 1024 ** 2).toFixed(1)} MiB`; +const time = (value) => value ? new Date(value).toLocaleString() : 'Unavailable'; +const age = (now, value) => value ? `${Math.max(0, (now - value) / 1000).toFixed(1)} s` : 'Unavailable'; + +function Row({ label, children }) { + return
+
{label}
+
{children}
+
; +} + +function Group({ title, children }) { + return
+

{title}

{children} +
; +} + +export default function VipRoverLearningCard({ roverId }) { + const socket = useSocket(); + const [snapshot, setSnapshot] = useState(null); + const [selectedId, setSelectedId] = useState(''); + const [working, setWorking] = useState(false); + const [error, setError] = useState(''); + const { value: settings, save } = useSettingsNamespace('roverLearning', { favorites: [] }); + const favorites = Array.isArray(settings?.favorites) ? settings.favorites : []; + + useEffect(() => { + let active = true; + const receive = (state) => { if (active) setSnapshot(state); }; + const subscribe = () => { + if (!socket.connected) return; + socket.timeout(5000).emit('roverLearning:subscribe', { roverId }, (failure, response) => { + if (!failure && response?.state) receive(response.state); + }); + }; + const disconnected = () => receive(null); + socket.on('roverLearning:state', receive); + socket.on('connect', subscribe); + socket.on('disconnect', disconnected); + subscribe(); + // A config enable can mount this card before async service startup has + // attached handlers. Resubscribe periodically without another data API. + const timer = setInterval(subscribe, 10000); + return () => { + active = false; + clearInterval(timer); + socket.off('roverLearning:state', receive); + socket.off('connect', subscribe); + socket.off('disconnect', disconnected); + socket.emit('roverLearning:unsubscribe'); + }; + }, [socket, roverId]); + + const state = snapshot?.roverId === roverId ? snapshot : null; + const models = [...(state?.models || [])].sort((a, b) => Number(favorites.includes(b.id)) - Number(favorites.includes(a.id)) || b.createdAt - a.createdAt); + const chosen = models.find((model) => model.id === selectedId) || models.find((model) => model.compatible) || models[0]; + const session = state?.session; + const training = state?.training; + const recording = state?.recording; + const limits = state?.limits; + const now = state?.updatedAt || 0; + + const act = (action) => { + setWorking(true); + setError(''); + socket.timeout(10000).emit('roverLearning:action', { action, roverId, modelId: chosen?.id }, (failure, response) => { + setWorking(false); + if (failure || response?.error) setError(response?.error || 'The server did not respond. Check the controller status before trying again.'); + else if (response?.state) setSnapshot(response.state); + }); + }; + const favorite = () => { + if (!chosen) return; + save((current) => { + const ids = Array.isArray(current?.favorites) ? current.favorites : []; + return { ...current, favorites: ids.includes(chosen.id) ? ids.filter((id) => id !== chosen.id) : [...ids, chosen.id] }; + }); + }; + + return +
+

+ {session ? `${session.modelName} · ${session.status}` : state ? 'Choose a model to control your rover.' : 'Waiting for rover learning status…'} +

+
+ + +
+
+ + +
+ {state?.control?.reason &&

{state.control.reason}

} +

Manual controls stay available and do not stop the model.

+

Recording: {recording ? recording.paused ? 'paused for disk space' : `${recording.sessions.length} human session(s)` : 'unavailable'} · Training: {training?.status || 'unavailable'}

+ {(error || state?.catalogError) &&

{error || state.catalogError}

} +
+ System details +
+
+ {bytes(recording?.bytes)} + {limits ? `${limits.recording.maxGiB} GiB` : 'Unavailable'} + {bytes(recording?.freeBytes)} + {limits ? `${limits.recording.minimumFreeGiB} GiB` : 'Unavailable'} + {limits ? `${limits.recording.sessionSeconds} s` : 'Unavailable'} + {bytes(recording?.pendingBytes)} + {display(recording?.droppedPending)} +
{recording?.sessions.map((item) =>
+

{item.roverId} · {item.video} · {age(now, item.startedAt)}

+

Dropped events: {item.droppedEvents}

{item.error &&

{item.error}

} +
)} +
+ {training?.status || 'Unavailable'} + {display(training?.steps)} + {display(training?.examples)} + {training?.loss == null ? 'Unavailable' : training.loss.toFixed(4)} + {training?.status === 'training' ? age(now, training.startedAt) : 'Not running'} + {time(training?.nextAttemptAt)} + {training ? `${training.visitedSessions} / ${training.skippedSessions}` : 'Unavailable'} + {display(training?.leasedSessions)} + {training?.lastAvailableWindows == null ? 'Unavailable' : `${training.lastAvailableWindows} / ${training.lastSampledWindows}`} + {limits ? `${limits.training.threads} / ${limits.training.batchSize}` : 'Unavailable'} + {limits ? `${limits.training.maxSamples} / ${limits.training.passesPerJob}` : 'Unavailable'} + {display(limits?.training.checkpointEverySteps)} +
{training?.error &&

{training.error}

} + {training?.reason &&

{training.reason}

} + {training?.rejectionReasons?.map((reason) =>

{reason}

)} + +
+ {chosen?.name || 'None'} + {time(chosen?.createdAt)} + {bytes(chosen?.weightBytes)} + {chosen?.parameters?.toLocaleString() || 'Unavailable'} + {chosen ? `${chosen.trainingSteps} / ${chosen.examplesProcessed}` : 'Unavailable'} + {display(chosen?.controls)} + {chosen?.evaluation || 'Unavailable'} + {models.length} +

Models are kept permanently. Favorites are saved in this browser.

 +
+ {session?.status || 'Stopped'} + {session?.startedBy || 'Unavailable'} + {age(now, session?.startedAt)} + {session?.latencyMs == null ? 'Unavailable' : `${session.latencyMs.toFixed(1)} ms`} + {age(now, session?.frameAt)} + {age(now, session?.sensorAt)} + {session?.lastResult || 'Unavailable'} + {display(limits?.driving.threads)} +
{session?.lastAction && 
{JSON.stringify(session.lastAction.command, null, 2)}
} + {session?.stopReason &&

{session.stopReason}

} + +
+
    + {state?.activity?.map((entry, index) =>
  • + {time(entry.at)} · {entry.message}{entry.roverId ? ` · ${entry.roverId}` : ''}{entry.detail ? ` · ${entry.detail}` : ''} +
    • )} +
+
+
+
+
+ ; +}