mirror of
https://github.com/legop3/MultiRoombaRover.git
synced 2026-11-14 18:58:49 -05:00
Merge pull request #30 from legop3/FEAT-rover-learner-model
Feat rover learner model
This commit is contained in:
legop3
@@ -11,6 +11,7 @@ const logger = require('../../globals/logger').child('commandService');
|
||||
const { isHeadlightBlocked } = require('../../rewards/definitions/darkness');
|
||||
const homeAssistantService = require('../homeAssistantService');
|
||||
const overcurrentProtectionService = require('../overcurrentProtectionService');
|
||||
const { subscribeCommandRecording, observeCommand, recordCommandRequest } = require('./recording');
|
||||
|
||||
// Command observations are intentionally separate from the global event bus.
|
||||
// Drive and motor commands can run at control-loop frequency, and publishing
|
||||
@@ -96,6 +97,7 @@ function issueCommand(roverId, payload) {
|
||||
const normalizedPayload = normalizeOutboundCommandPayload(payload);
|
||||
const message = { ...normalizedPayload, id };
|
||||
record.ws.send(JSON.stringify(message));
|
||||
observeCommand({ phase: 'dispatched', roverId, commandId: id, command: normalizedPayload });
|
||||
pendingCommands.set(id, { roverId, ts: Date.now(), type: normalizedPayload.type });
|
||||
logger.info('Issued command', roverId, normalizedPayload.type, id);
|
||||
return id;
|
||||
@@ -130,6 +132,7 @@ function handleAck(msg) {
|
||||
const pending = pendingCommands.get(msg.id);
|
||||
if (!pending) return;
|
||||
pendingCommands.delete(msg.id);
|
||||
observeCommand({ phase: 'acknowledged', roverId: pending.roverId, commandId: msg.id, acknowledgement: msg });
|
||||
logger.info('Command acknowledged', pending.roverId, pending.type, msg.status);
|
||||
io.emit('commandAck', {
|
||||
roverId: pending.roverId,
|
||||
@@ -255,145 +258,149 @@ module.exports = {
|
||||
getRecentDriveActivity,
|
||||
setDriveCooldown,
|
||||
commandEvents,
|
||||
subscribeCommandRecording,
|
||||
submitCommand,
|
||||
};
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
function handleCommand({ roverId, type, data } = {}, cb) {
|
||||
const reply = typeof cb === 'function' ? cb : () => {};
|
||||
try {
|
||||
if (!roverId) {
|
||||
throw new Error('roverId required');
|
||||
// Shared authorization/safety boundary for browser and service-owned control.
|
||||
function submitCommand(socket, { roverId, type, data } = {}, cb) {
|
||||
const reply = typeof cb === 'function' ? cb : () => {};
|
||||
try {
|
||||
if (!roverId) {
|
||||
throw new Error('roverId required');
|
||||
}
|
||||
if (!type) {
|
||||
throw new Error('type required');
|
||||
}
|
||||
if (type === 'audioLevels') {
|
||||
throw new Error('audioLevels command is service-managed');
|
||||
}
|
||||
let payload = data ? { ...data } : {};
|
||||
if (type === 'headlight' && isHeadlightBlocked()) {
|
||||
logger.info('Ignoring headlight command while darkness lock is active', { socketId: socket.id, roverId });
|
||||
reply({ ignored: true, reason: 'darknessActive' });
|
||||
return;
|
||||
}
|
||||
if (type === 'laser' && isLaserCommandBlockedByRoomLightLock(payload)) {
|
||||
logger.info('Ignoring laser command while room lights are locked on', {
|
||||
socketId: socket.id,
|
||||
roverId,
|
||||
action: getLaserAction(payload),
|
||||
});
|
||||
reply({ ignored: true, reason: 'roomLightsLockedOn' });
|
||||
return;
|
||||
}
|
||||
const isRebootCommand = type === 'reboot';
|
||||
const isUpdateCommand = type === 'update';
|
||||
const isSongCommand = type === 'song' || (type === 'raw' && isSongRawPayload(payload));
|
||||
const isAdminSocket = isAdmin(socket);
|
||||
if (!isAdminSocket && isDeterred(socket)) {
|
||||
throw new Error('Not authorized');
|
||||
}
|
||||
/*
|
||||
Both structured song commands and raw Open Interface song payloads
|
||||
reach this shared flag. Enforcing mute here covers the VIP MIDI beeper
|
||||
and any future browser beeper without affecting unrelated driving.
|
||||
*/
|
||||
if (!isAdminSocket && isSongCommand && isMuted(socket)) {
|
||||
throw new Error('Muted');
|
||||
}
|
||||
// Rover updates run a privileged, root-owned helper on the Pi. Keep this
|
||||
// in the same explicit admin-only branch as reboot instead of relying on
|
||||
// drive ownership checks, because having a turn should not grant system
|
||||
// maintenance privileges.
|
||||
if ((isRebootCommand || isUpdateCommand) && !isAdminSocket) {
|
||||
throw new Error('Not authorized');
|
||||
}
|
||||
if (!isSongCommand && !isRebootCommand && !isUpdateCommand && !roverManager.canDrive(roverId, socket)) {
|
||||
throw new Error('Not your turn or no control');
|
||||
}
|
||||
const driveDirect = payload?.driveDirect;
|
||||
if (type === 'drive' && driveDirect) {
|
||||
if (!isAdminSocket) {
|
||||
const safeDrive = roverManager.applyPrivateDriveSafety(roverId, socket, driveDirect);
|
||||
if (safeDrive) {
|
||||
payload.driveDirect = safeDrive;
|
||||
}
|
||||
}
|
||||
if (!type) {
|
||||
throw new Error('type required');
|
||||
}
|
||||
if (type === 'audioLevels') {
|
||||
throw new Error('audioLevels command is service-managed');
|
||||
}
|
||||
let payload = data ? { ...data } : {};
|
||||
if (type === 'headlight' && isHeadlightBlocked()) {
|
||||
logger.info('Ignoring headlight command while darkness lock is active', { socketId: socket.id, roverId });
|
||||
reply({ ignored: true, reason: 'darknessActive' });
|
||||
return;
|
||||
const left = Number(payload?.driveDirect?.left);
|
||||
const right = Number(payload?.driveDirect?.right);
|
||||
const speed = Math.max(Math.abs(left), Math.abs(right));
|
||||
if (!isAdminSocket) {
|
||||
const blockedUntil = driveCooldowns.get(roverId);
|
||||
if (blockedUntil && Date.now() < blockedUntil && speed > 0) {
|
||||
const reason = isLockdownAdmin(socket)
|
||||
? 'Drive blocked: cooldown'
|
||||
: 'Drive blocked: safety cooldown';
|
||||
throw new Error(reason);
|
||||
}
|
||||
}
|
||||
if (type === 'laser' && isLaserCommandBlockedByRoomLightLock(payload)) {
|
||||
logger.info('Ignoring laser command while room lights are locked on', {
|
||||
if (speed > 0) {
|
||||
let direction = 'turn';
|
||||
if (left > 0 && right > 0) direction = 'forward';
|
||||
if (left < 0 && right < 0) direction = 'backward';
|
||||
lastDriveActivity.set(roverId, {
|
||||
ts: Date.now(),
|
||||
socketId: socket.id,
|
||||
roverId,
|
||||
action: getLaserAction(payload),
|
||||
});
|
||||
reply({ ignored: true, reason: 'roomLightsLockedOn' });
|
||||
return;
|
||||
}
|
||||
const isRebootCommand = type === 'reboot';
|
||||
const isUpdateCommand = type === 'update';
|
||||
const isSongCommand = type === 'song' || (type === 'raw' && isSongRawPayload(payload));
|
||||
const isAdminSocket = isAdmin(socket);
|
||||
if (!isAdminSocket && isDeterred(socket)) {
|
||||
throw new Error('Not authorized');
|
||||
}
|
||||
/*
|
||||
Both structured song commands and raw Open Interface song payloads
|
||||
reach this shared flag. Enforcing mute here covers the VIP MIDI beeper
|
||||
and any future browser beeper without affecting unrelated driving.
|
||||
*/
|
||||
if (!isAdminSocket && isSongCommand && isMuted(socket)) {
|
||||
throw new Error('Muted');
|
||||
}
|
||||
// Rover updates run a privileged, root-owned helper on the Pi. Keep this
|
||||
// in the same explicit admin-only branch as reboot instead of relying on
|
||||
// drive ownership checks, because having a turn should not grant system
|
||||
// maintenance privileges.
|
||||
if ((isRebootCommand || isUpdateCommand) && !isAdminSocket) {
|
||||
throw new Error('Not authorized');
|
||||
}
|
||||
if (!isSongCommand && !isRebootCommand && !isUpdateCommand && !roverManager.canDrive(roverId, socket)) {
|
||||
throw new Error('Not your turn or no control');
|
||||
}
|
||||
const driveDirect = payload?.driveDirect;
|
||||
if (type === 'drive' && driveDirect) {
|
||||
if (!isAdminSocket) {
|
||||
const safeDrive = roverManager.applyPrivateDriveSafety(roverId, socket, driveDirect);
|
||||
if (safeDrive) {
|
||||
payload.driveDirect = safeDrive;
|
||||
}
|
||||
}
|
||||
const left = Number(payload?.driveDirect?.left);
|
||||
const right = Number(payload?.driveDirect?.right);
|
||||
const speed = Math.max(Math.abs(left), Math.abs(right));
|
||||
if (!isAdminSocket) {
|
||||
const blockedUntil = driveCooldowns.get(roverId);
|
||||
if (blockedUntil && Date.now() < blockedUntil && speed > 0) {
|
||||
const reason = isLockdownAdmin(socket)
|
||||
? 'Drive blocked: cooldown'
|
||||
: 'Drive blocked: safety cooldown';
|
||||
throw new Error(reason);
|
||||
}
|
||||
}
|
||||
if (speed > 0) {
|
||||
let direction = 'turn';
|
||||
if (left > 0 && right > 0) direction = 'forward';
|
||||
if (left < 0 && right < 0) direction = 'backward';
|
||||
lastDriveActivity.set(roverId, {
|
||||
ts: Date.now(),
|
||||
socketId: socket.id,
|
||||
direction,
|
||||
speed,
|
||||
isAdmin: isAdminSocket,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (type === 'drive' || type === 'motors') {
|
||||
/*
|
||||
Role is supplied at the command boundary because telemetry does not
|
||||
identify the operator who produced the active motor intent. Admin and
|
||||
lockdown commands therefore enter the service explicitly bypassed;
|
||||
they are recorded for status visibility but are never scaled, blocked,
|
||||
or countermanded by a later sensor frame.
|
||||
*/
|
||||
payload = overcurrentProtectionService.protectCommand(roverId, type, payload, {
|
||||
bypassed: isAdminSocket,
|
||||
direction,
|
||||
speed,
|
||||
isAdmin: isAdminSocket,
|
||||
});
|
||||
}
|
||||
const id = issueCommand(roverId, { type, ...payload });
|
||||
commandEvents.emit('observation', {
|
||||
ts: Date.now(),
|
||||
roverId: String(roverId),
|
||||
type,
|
||||
commandId: id,
|
||||
outcome: 'issued',
|
||||
socketId: socket.id,
|
||||
// Payloads are omitted deliberately: raw OI, TTS, and maintenance
|
||||
// commands can carry arbitrary content. Their structured type/outcome
|
||||
// supplies analytics without accidentally persisting secret material.
|
||||
});
|
||||
logger.info('Queued command', socket.id, roverId, type);
|
||||
if (shouldRecordTurnActivity(type, payload)) {
|
||||
try {
|
||||
const { recordActivity } = require('../turnService');
|
||||
recordActivity(roverId, socket.id);
|
||||
} catch (err) {
|
||||
// Activity recording is best-effort because command delivery should not fail if turn bookkeeping has a transient issue.
|
||||
}
|
||||
}
|
||||
reply({ id });
|
||||
} catch (err) {
|
||||
logger.warn('Command rejected', socket.id, err.message);
|
||||
commandEvents.emit('observation', {
|
||||
ts: Date.now(),
|
||||
roverId: roverId ? String(roverId) : null,
|
||||
type: type || 'unknown',
|
||||
outcome: 'rejected',
|
||||
socketId: socket.id,
|
||||
error: err.message,
|
||||
});
|
||||
reply({ error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
socket.on('command', handleCommand);
|
||||
socket.on('command:issue', handleCommand);
|
||||
if (type === 'drive' || type === 'motors') {
|
||||
/*
|
||||
Role is supplied at the command boundary because telemetry does not
|
||||
identify the operator who produced the active motor intent. Admin and
|
||||
lockdown commands therefore enter the service explicitly bypassed;
|
||||
they are recorded for status visibility but are never scaled, blocked,
|
||||
or countermanded by a later sensor frame.
|
||||
*/
|
||||
payload = overcurrentProtectionService.protectCommand(roverId, type, payload, {
|
||||
bypassed: isAdminSocket,
|
||||
});
|
||||
}
|
||||
const id = issueCommand(roverId, { type, ...payload });
|
||||
commandEvents.emit('observation', {
|
||||
ts: Date.now(),
|
||||
roverId: String(roverId),
|
||||
type,
|
||||
commandId: id,
|
||||
outcome: 'issued',
|
||||
socketId: socket.id,
|
||||
// Payloads are omitted deliberately: raw OI, TTS, and maintenance
|
||||
// commands can carry arbitrary content. Their structured type/outcome
|
||||
// supplies analytics without accidentally persisting secret material.
|
||||
});
|
||||
logger.info('Queued command', socket.id, roverId, type);
|
||||
if (shouldRecordTurnActivity(type, payload)) {
|
||||
try {
|
||||
const { recordActivity } = require('../turnService');
|
||||
recordActivity(roverId, socket.id);
|
||||
} catch (err) {
|
||||
// Activity recording is best-effort because command delivery should not fail if turn bookkeeping has a transient issue.
|
||||
}
|
||||
}
|
||||
reply({ id });
|
||||
} catch (err) {
|
||||
logger.warn('Command rejected', socket.id, err.message);
|
||||
commandEvents.emit('observation', {
|
||||
ts: Date.now(),
|
||||
roverId: roverId ? String(roverId) : null,
|
||||
type: type || 'unknown',
|
||||
outcome: 'rejected',
|
||||
socketId: socket.id,
|
||||
error: err.message,
|
||||
});
|
||||
reply({ error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
const recordedCommand = (request, reply) => recordCommandRequest(socket, request, reply, (message, cb) => submitCommand(socket, message, cb));
|
||||
socket.on('command', recordedCommand);
|
||||
socket.on('command:issue', recordedCommand);
|
||||
|
||||
socket.on('command:updateAllRovers', (_payload = {}, cb) => {
|
||||
const reply = typeof cb === 'function' ? cb : () => {};
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Optional detailed command observation, separate from payload-free analytics.
|
||||
// Subscribers must only enqueue work; recorder failures cannot fail a command.
|
||||
const { AsyncLocalStorage } = require('node:async_hooks');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
|
||||
const observers = new Set();
|
||||
const requests = new AsyncLocalStorage();
|
||||
|
||||
function subscribeCommandRecording(observer) {
|
||||
observers.add(observer);
|
||||
return () => observers.delete(observer);
|
||||
}
|
||||
|
||||
function observeCommand(event) {
|
||||
if (!observers.size) return;
|
||||
const scope = requests.getStore();
|
||||
const context = scope?.active ? scope.context : null;
|
||||
for (const observer of observers) {
|
||||
try {
|
||||
// Snapshot before safety code or a caller mutates a nested payload. Each
|
||||
// subscriber receives its own copy and cannot change command delivery.
|
||||
observer(structuredClone({
|
||||
ts: Date.now(),
|
||||
monotonicNs: process.hrtime.bigint().toString(),
|
||||
source: context ? 'client' : 'server',
|
||||
...context,
|
||||
...event,
|
||||
}));
|
||||
} catch {
|
||||
// Recording is deliberately best-effort at this boundary. The owning
|
||||
// recorder reports its storage health independently of driver replies.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function recordCommandRequest(socket, request, reply, execute, source = 'client') {
|
||||
if (!observers.size) return execute(request, reply);
|
||||
const context = { requestId: randomUUID(), socketId: socket.id, userId: socket.data?.userId || null, source };
|
||||
const scope = { active: true, context };
|
||||
return requests.run(scope, () => {
|
||||
try {
|
||||
observeCommand({ phase: 'requested', roverId: request?.roverId, command: request });
|
||||
return execute(request, (result) => {
|
||||
observeCommand({ phase: 'result', roverId: request?.roverId, result });
|
||||
if (typeof reply === 'function') reply(result);
|
||||
});
|
||||
} finally {
|
||||
// The command handler is synchronous. A safety timer created inside it
|
||||
// inherits async context but its later commands are server actions.
|
||||
scope.active = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { subscribeCommandRecording, observeCommand, recordCommandRequest };
|
||||
@@ -0,0 +1,15 @@
|
||||
// Match actuator configuration, not a rover's display name or changing telemetry.
|
||||
const { isDeepStrictEqual } = require('node:util');
|
||||
|
||||
function controlProfile(meta = {}) {
|
||||
return Object.fromEntries(['cameraServo', 'peripherals', 'headlight', 'laser', 'roomba', 'platform']
|
||||
.map((key) => [key, meta?.[key] ?? null]));
|
||||
}
|
||||
|
||||
function compatible(model, record) {
|
||||
if (model?.specification?.version !== 2 || !record) return false;
|
||||
const profile = controlProfile(record.meta);
|
||||
return Array.isArray(model.controlProfiles) && model.controlProfiles.some((saved) => isDeepStrictEqual(saved, profile));
|
||||
}
|
||||
|
||||
module.exports = { controlProfile, compatible };
|
||||
@@ -0,0 +1,47 @@
|
||||
// Metadata only: loading the configuration must never start the learning runtime.
|
||||
const { strictObject, boolean, integer, number, string } = require('../../configuration/schemaHelpers');
|
||||
|
||||
module.exports = {
|
||||
key: 'roverLearning',
|
||||
feature: true,
|
||||
defaultValue: {
|
||||
enabled: false,
|
||||
recording: { maxGiB: 20, minimumFreeGiB: 2, sessionSeconds: 300 },
|
||||
driving: { threads: 4, actionThreshold: 0.7, staleMs: 2000 },
|
||||
training: {
|
||||
enabled: true, python: '/opt/rover-learning/bin/python', threads: 8,
|
||||
batchSize: 16, maxSamples: 1024, minimumSamples: 128, passesPerJob: 2,
|
||||
maxStepsPerJob: 200, checkpointEverySteps: 1000,
|
||||
minimumDrivingMinutes: 30, newDrivingMinutesPerModel: 10,
|
||||
intervalSeconds: 60, maxJobSeconds: 600,
|
||||
},
|
||||
},
|
||||
schema: strictObject({
|
||||
enabled: boolean({ description: 'Enable human-session recording, background learning, and verified-user model control. Disabled starts no workers or storage.' }),
|
||||
driving: strictObject({
|
||||
threads: integer({ minimum: 1, maximum: 16, description: 'CPU threads per active model controller.' }),
|
||||
actionThreshold: number({ minimum: 0.5, maximum: 0.99, description: 'Minimum predicted action probability for issuing a new control event.' }),
|
||||
staleMs: integer({ minimum: 1000, maximum: 5000, description: 'Stop when video, sensors, or model predictions are older than this many milliseconds.' }),
|
||||
}, { description: 'Live model execution. Manual inputs do not stop or pause the model.', required: ['threads', 'actionThreshold', 'staleMs'] }),
|
||||
recording: strictObject({
|
||||
maxGiB: number({ minimum: 0.1, maximum: 100000, description: 'Rolling recording budget in GiB. Checked every 15 seconds; active files can temporarily exceed it. Published models are never pruned.' }),
|
||||
minimumFreeGiB: number({ minimum: 0.1, maximum: 100000, description: 'Pause recording below this amount of available filesystem space, in GiB.' }),
|
||||
sessionSeconds: integer({ minimum: 30, maximum: 3600, description: 'Rotate video and telemetry sessions after this many seconds so completed sessions can be pruned.' }),
|
||||
}, { description: 'Bounded recording storage. Limits never delete published models.', required: ['maxGiB', 'minimumFreeGiB', 'sessionSeconds'] }),
|
||||
training: strictObject({
|
||||
enabled: boolean({ description: 'Train periodically from completed usable human recordings while the service is enabled.' }),
|
||||
python: string({ minLength: 1, description: 'Python executable with the service requirements installed. The container supplies /opt/rover-learning/bin/python.' }),
|
||||
threads: integer({ minimum: 1, maximum: 32, description: 'CPU threads for the lower-priority training process.' }),
|
||||
batchSize: integer({ minimum: 1, maximum: 64, description: 'Training windows per optimizer step.' }),
|
||||
maxSamples: integer({ minimum: 128, maximum: 2048, description: 'Maximum decoded windows retained in RAM per job. Each contains four 160x120 RGB frames.' }),
|
||||
minimumSamples: integer({ minimum: 1, maximum: 128, description: 'Usable sampled windows required before training.' }),
|
||||
passesPerJob: integer({ minimum: 1, maximum: 10, description: 'Shuffled passes over the sampled windows per job.' }),
|
||||
minimumDrivingMinutes: number({ minimum: 1, maximum: 100000, description: 'Minimum distinct usable training footage in minutes before promoting Latest; repeated passes do not count.' }),
|
||||
newDrivingMinutesPerModel: number({ minimum: 1, maximum: 100000, description: 'Additional distinct usable footage required between Latest promotions.' }),
|
||||
maxStepsPerJob: integer({ minimum: 1, maximum: 10000, description: 'Optimizer step cap per finite training job.' }),
|
||||
checkpointEverySteps: integer({ minimum: 1, maximum: 100000, description: 'Accumulated training steps between gated Latest promotions. Permanent named snapshots are created by administrators.' }),
|
||||
intervalSeconds: integer({ minimum: 30, maximum: 86400, description: 'Minimum seconds between completed training jobs.' }),
|
||||
maxJobSeconds: integer({ minimum: 30, maximum: 3600, description: 'Wall-clock budget including video decoding; slow/stuck workers are stopped.' }),
|
||||
}, { description: 'Ongoing CPU learning, sampling, and gated Latest checkpoint updates.', required: ['enabled', 'python', 'threads', 'batchSize', 'maxSamples', 'minimumSamples', 'passesPerJob', 'minimumDrivingMinutes', 'newDrivingMinutesPerModel', 'maxStepsPerJob', 'checkpointEverySteps', 'intervalSeconds', 'maxJobSeconds'] }),
|
||||
}, { description: 'Optional rover recording and learning service.', required: ['enabled', 'recording', 'training', 'driving'] }),
|
||||
};
|
||||
@@ -0,0 +1,60 @@
|
||||
// Export an explicit allowlist, never the raw socket/session/configuration object.
|
||||
const os = require('node:os');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { createHash } = require('node:crypto');
|
||||
const applicationVersion = require('../../../package.json').version;
|
||||
const hash = createHash('sha256');
|
||||
for (const file of ['workers/policy.py', 'workers/dataset.py', 'workers/train.py', 'workers/infer.py',
|
||||
'models.js', 'driving.js', 'recording.js', 'training.js', 'socketGateway.js', 'diagnostics.js']) {
|
||||
hash.update(fs.readFileSync(path.join(__dirname, file)));
|
||||
}
|
||||
const implementationSha256 = hash.digest('hex');
|
||||
|
||||
function errorCategory(value) {
|
||||
if (!value) return null;
|
||||
const text = String(value).toLowerCase();
|
||||
for (const [pattern, category] of [
|
||||
[/timestamp|pts|clock|aligned/, 'time_alignment'], [/sensor/, 'sensors'],
|
||||
[/video|decode|ffmpeg|rtsp/, 'video'], [/permission|turn|control permission/, 'permission'],
|
||||
[/schema|shape|version/, 'model_schema'], [/space|disk|enospc/, 'storage'],
|
||||
[/sample|window|command/, 'dataset_or_command'], [/worker|exited|process/, 'worker'],
|
||||
]) if (pattern.test(text)) return category;
|
||||
return 'unclassified';
|
||||
}
|
||||
|
||||
function report(state, controller) {
|
||||
const training = state.training || {};
|
||||
const metrics = training.diagnostics || {};
|
||||
const session = state.session;
|
||||
return {
|
||||
reportVersion: 1, generatedAtUnixMs: state.updatedAt,
|
||||
software: { applicationVersion, implementationSha256, policyVersion: 2, node: process.version,
|
||||
dependencies: metrics.dependencies || null },
|
||||
host: { logicalCpus: os.cpus().length, totalMemoryBytes: os.totalmem(), freeMemoryBytes: os.freemem(), loadAverage: os.loadavg() },
|
||||
settings: state.limits,
|
||||
recording: { paused: state.recording.paused, bytes: state.recording.bytes,
|
||||
freeBytes: state.recording.freeBytes, pendingBytes: state.recording.pendingBytes,
|
||||
droppedPending: state.recording.droppedPending,
|
||||
sessions: state.recording.sessions.map((item) => ({ startedAtUnixMs: item.startedAt,
|
||||
status: item.video, droppedEvents: item.droppedEvents, errorCategory: errorCategory(item.error) })) },
|
||||
training: { status: training.status, steps: training.steps, examples: training.examples,
|
||||
loss: training.loss, losses: training.losses || null, publication: training.publication || null,
|
||||
evaluation: training.evaluation || null, dataset: metrics.dataset || training.dataset || null,
|
||||
resources: { decodeSeconds: metrics.decodeSeconds ?? null, trainingSeconds: metrics.trainingSeconds ?? null,
|
||||
evaluationSeconds: metrics.evaluationSeconds ?? null, checkpointSeconds: metrics.checkpointSeconds ?? null, cpuSeconds: metrics.cpuSeconds ?? null,
|
||||
peakRssMiB: metrics.peakRssMiB ?? null },
|
||||
history: training.history || [], rejectionCategories: (training.rejectionReasons || []).map(errorCategory),
|
||||
errorCategory: errorCategory(training.error), nextAttemptAtUnixMs: training.nextAttemptAt },
|
||||
model: session ? { id: session.modelId, checkpointId: session.checkpointId, threshold: session.threshold, appliedThreshold: session.appliedThreshold } : null,
|
||||
controller: controller ? { modelId: controller.modelId, sensorPresent: controller.sensorPresent || null,
|
||||
stopCategory: errorCategory(controller.stopReason), history: (controller.history || []).map((item) => ({
|
||||
...item, submissions: item.submissions.map((submission) => ({ type: submission.type,
|
||||
wheels: submission.wheels, result: submission.result === 'issued' ? 'issued' : errorCategory(submission.result) })) })) } : null,
|
||||
sensorFields: metrics.policy?.sensorFields || null,
|
||||
timing: { alignment: 'server_receipt', browserVideoLatencyMs: null, wheelCommandExpiryMs: null },
|
||||
events: state.activity.map(({ at, message }) => ({ atUnixMs: at, message })),
|
||||
unavailable: ['browser_video_latency', 'actual_physical_command_execution', 'live_training_cpu_utilization'],
|
||||
};
|
||||
}
|
||||
module.exports = { report };
|
||||
@@ -0,0 +1,208 @@
|
||||
// Autonomy owns a live user's control context, never an administrative identity.
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
const roverManager = require('../roverManager');
|
||||
const { isVerified, isDeterred } = require('../verificationService');
|
||||
const { isAdmin } = require('../roleService');
|
||||
const { submitCommand, issueCommand, subscribeCommandRecording } = require('../commandService');
|
||||
const { recordCommandRequest } = require('../commandService/recording');
|
||||
const { acquireModel } = require('./models');
|
||||
const { compatible } = require('./capabilities');
|
||||
|
||||
const OPERATING = new Set(['drive', 'motors', 'servo', 'headlight', 'laser', 'horn', 'peripheral', 'song', 'raw']);
|
||||
const clamp = (value, min, max) => Math.max(min, Math.min(max, Number(value) || 0));
|
||||
|
||||
function canControl(socket, roverId) {
|
||||
return Boolean(socket?.connected && isVerified(socket) && (isAdmin(socket) || !isDeterred(socket))
|
||||
&& roverManager.rovers.get(roverId)?.ws && roverManager.canDrive(roverId, socket));
|
||||
}
|
||||
|
||||
function createDriving({ root, config, onChange, activity }) {
|
||||
const sessions = new Map();
|
||||
const recent = new Map();
|
||||
let stopping = false;
|
||||
let watchdog;
|
||||
let unsubscribe;
|
||||
|
||||
function sendState(session, event) {
|
||||
const input = session.worker.stdin;
|
||||
if (input.destroyed || !input.writable || input.writableLength > 256 * 1024) return;
|
||||
input.write(JSON.stringify(event) + '\n');
|
||||
}
|
||||
|
||||
function transmit(session, command) {
|
||||
if (session.status === 'stopping') return;
|
||||
if (!canControl(session.socket, session.roverId)) { stop(session.roverId, 'Control permission lost'); return; }
|
||||
if (!OPERATING.has(command?.type)) throw new Error('Unsupported model command');
|
||||
const record = roverManager.rovers.get(session.roverId);
|
||||
if (command.type === 'drive') {
|
||||
const limit = Math.min(500, Number(record.meta?.maxWheelSpeed) || 500);
|
||||
command.driveDirect = { left: Math.round(clamp(command.driveDirect?.left, -limit, limit)), right: Math.round(clamp(command.driveDirect?.right, -limit, limit)) };
|
||||
}
|
||||
if (command.type === 'motors') {
|
||||
command.motorPwm = { main: clamp(command.motorPwm?.main, -127, 127), side: clamp(command.motorPwm?.side, -127, 127), vacuum: clamp(command.motorPwm?.vacuum, 0, 127) };
|
||||
}
|
||||
if (command.type === 'servo') {
|
||||
const servo = record.meta?.cameraServo;
|
||||
if (!servo?.enabled) { session.submissions.push({ command, result: 'Servo unavailable' }); return; }
|
||||
command.servo.angle = clamp(command.servo?.angle, servo.minAngle ?? 0, servo.maxAngle ?? 180);
|
||||
}
|
||||
const { type, ...data } = command;
|
||||
recordCommandRequest(session.socket, { roverId: session.roverId, type, data }, (result) => {
|
||||
session.lastResult = result.error || result.reason || 'issued';
|
||||
session.submissions.push({ command, result: session.lastResult });
|
||||
if (result.id) session.lastAction = { at: Date.now(), command };
|
||||
}, (request, reply) => submitCommand(session.socket, request, reply), 'model');
|
||||
}
|
||||
|
||||
function stop(roverId, reason = 'Stopped by user') {
|
||||
const session = sessions.get(roverId);
|
||||
if (!session) return Promise.resolve();
|
||||
if (session.done) return session.done;
|
||||
session.status = 'stopping';
|
||||
session.stopReason = reason;
|
||||
// A stop must still reach the rover after turn permission is revoked.
|
||||
for (const command of [
|
||||
{ type: 'drive', driveDirect: { left: 0, right: 0 } },
|
||||
{ type: 'motors', motorPwm: { main: 0, side: 0, vacuum: 0 } },
|
||||
{ type: 'horn', horn: { action: 'stop' } },
|
||||
]) { try { issueCommand(roverId, command); } catch { /* Rover may already be offline. */ } }
|
||||
session.worker.kill('SIGTERM');
|
||||
const timer = setTimeout(() => session.worker.kill('SIGKILL'), 2000);
|
||||
session.done = session.closed.then(() => {
|
||||
clearTimeout(timer);
|
||||
recent.delete(roverId);
|
||||
recent.set(roverId, { modelId: session.modelId, stopReason: reason, history: session.history });
|
||||
if (recent.size > 20) recent.delete(recent.keys().next().value);
|
||||
sessions.delete(roverId);
|
||||
activity('Controller stopped', { roverId, reason });
|
||||
onChange();
|
||||
});
|
||||
onChange();
|
||||
return session.done;
|
||||
}
|
||||
|
||||
async function start(socket, roverId, modelId) {
|
||||
if (stopping || !canControl(socket, roverId)) throw new Error('You cannot control this rover');
|
||||
const acquired = await acquireModel(root, modelId);
|
||||
const { directory, metadata } = acquired;
|
||||
try {
|
||||
if (!compatible(metadata, roverManager.rovers.get(roverId))) throw new Error('Model is not compatible with this rover');
|
||||
if (stopping || !canControl(socket, roverId)) throw new Error('Control permission changed');
|
||||
if (sessions.has(roverId)) throw new Error('Stop the current controller before starting another model');
|
||||
} catch (error) { await acquired.release(); throw error; }
|
||||
const worker = spawn(config.training.python, [path.join(__dirname, 'workers/infer.py'), '--model', directory,
|
||||
'--url', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`, '--threads', String(config.driving.threads),
|
||||
'--threshold', String(config.driving.actionThreshold)], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } });
|
||||
const session = { worker, socket, roverId, modelId, checkpointId: metadata.id, modelName: modelId === 'latest' ? 'Latest' : modelId === 'previous' ? 'Previous' : metadata.name, status: 'starting', startedAt: Date.now(), lastPredictionAt: null, threshold: config.driving.actionThreshold, history: [] };
|
||||
session.closed = new Promise((resolve) => worker.once('close', resolve));
|
||||
session.closed.then(() => acquired.release()).catch((error) => activity('Checkpoint release failed', { error: error.message }));
|
||||
sessions.set(roverId, session);
|
||||
recent.delete(roverId);
|
||||
let output = '';
|
||||
let stderr = '';
|
||||
worker.stdin.on('error', () => { stop(roverId, 'Worker input closed'); });
|
||||
worker.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-1000); });
|
||||
worker.on('error', (error) => { stop(roverId, error.message); });
|
||||
worker.once('close', () => { stop(roverId, stderr || 'Worker exited'); });
|
||||
worker.stdout.on('data', (chunk) => {
|
||||
if (session.status === 'stopping') return;
|
||||
output += chunk;
|
||||
if (output.length > 1024 * 1024) { stop(roverId, 'Worker output exceeded limit'); return; }
|
||||
let newline;
|
||||
while ((newline = output.indexOf('\n')) >= 0) {
|
||||
const line = output.slice(0, newline);
|
||||
output = output.slice(newline + 1);
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
if (message.kind === 'error') throw new Error(message.message);
|
||||
if (message.kind !== 'prediction') continue;
|
||||
if (!Number.isFinite(message.frameAt) || Math.abs(Date.now() - message.frameAt) > config.driving.staleMs) throw new Error('Video is stale');
|
||||
if (!Number.isFinite(message.sensorAt) || Math.abs(Date.now() - message.sensorAt) > config.driving.staleMs) throw new Error('Sensors are stale');
|
||||
if (!canControl(socket, roverId)) throw new Error('Control permission lost');
|
||||
session.status = 'running';
|
||||
session.lastPredictionAt = Date.now();
|
||||
session.frameAt = message.frameAt;
|
||||
session.sensorAt = message.sensorAt;
|
||||
session.latencyMs = message.latencyMs;
|
||||
session.proposals = Array.isArray(message.proposals) ? message.proposals.slice(0, 64) : [];
|
||||
session.appliedThreshold = message.threshold;
|
||||
session.wheelSpeeds = message.wheelSpeeds;
|
||||
session.sensorPresent = message.sensorPresent;
|
||||
session.submissions = [];
|
||||
if (!Array.isArray(message.commands) || message.commands.length > 65) throw new Error('Invalid action batch');
|
||||
session.predictionStatus = message.wheelSpeeds?.some((value) => value !== 0) ? 'Sending wheel controls' : 'Predicting stopped wheels';
|
||||
// Manual commands intentionally do not stop or pause this controller.
|
||||
for (const command of message.commands) transmit(session, command);
|
||||
// Bound report history independently of camera and UI frame rates.
|
||||
if (!session.history.length || Date.now() - session.history.at(-1).at >= 500) {
|
||||
session.history.push({ at: Date.now(), wheelSpeeds: session.wheelSpeeds,
|
||||
latencyMs: session.latencyMs, frameAgeMs: Date.now() - session.frameAt,
|
||||
sensorAgeMs: Date.now() - session.sensorAt, threshold: session.appliedThreshold,
|
||||
proposals: [...session.proposals].sort((a, b) => b.score - a.score).slice(0, 8).map(({ slot, score, reason }) => ({ slot, score, reason })),
|
||||
submissions: session.submissions.slice(0, 16).map(({ command, result }) => ({ type: command.type, result,
|
||||
wheels: command.type === 'drive' ? command.driveDirect : undefined })) });
|
||||
if (session.history.length > 60) session.history.shift();
|
||||
}
|
||||
} catch (error) { stop(roverId, error.message); break; }
|
||||
}
|
||||
});
|
||||
const sensors = roverManager.rovers.get(roverId)?.lastSensor?.decoded;
|
||||
if (sensors) sendState(session, { kind: 'sensor', ts: Date.now(), data: sensors });
|
||||
activity('Controller started', { roverId, model: metadata.name, socketId: socket.id });
|
||||
onChange();
|
||||
}
|
||||
|
||||
function sensor(event) {
|
||||
const session = sessions.get(String(event.roverId));
|
||||
if (session) sendState(session, { kind: 'sensor', ts: Date.now(), data: event.sensors });
|
||||
}
|
||||
|
||||
function attach() {
|
||||
roverManager.managerEvents.on('sensor', sensor);
|
||||
unsubscribe = subscribeCommandRecording((event) => {
|
||||
const session = sessions.get(String(event.roverId));
|
||||
if (session && event.phase === 'dispatched') sendState(session, { kind: 'command', ts: event.ts, data: event.command });
|
||||
});
|
||||
watchdog = setInterval(() => {
|
||||
for (const session of sessions.values()) {
|
||||
if (!canControl(session.socket, session.roverId)) stop(session.roverId, 'Control permission lost');
|
||||
else if (Date.now() - (session.lastPredictionAt || session.startedAt) > (session.lastPredictionAt ? config.driving.staleMs : 30000)) {
|
||||
stop(session.roverId, 'No fresh model predictions');
|
||||
}
|
||||
}
|
||||
}, 250);
|
||||
}
|
||||
|
||||
return {
|
||||
attach, start, stop, canControl,
|
||||
setThreshold(socket, roverId, value) {
|
||||
const session = sessions.get(roverId);
|
||||
if (!session || session.status === 'stopping' || !canControl(socket, roverId)) throw new Error('You cannot adjust this controller');
|
||||
if (typeof value !== 'number' || !Number.isFinite(value) || value < .05 || value > .99) throw new Error('Threshold must be between 0.05 and 0.99');
|
||||
session.threshold = value;
|
||||
sendState(session, { kind: 'settings', ts: Date.now(), threshold: value });
|
||||
},
|
||||
getDiagnostics(id) {
|
||||
const session = sessions.get(id);
|
||||
return session ? { modelId: session.modelId, sensorPresent: session.sensorPresent, history: session.history } : recent.get(id) || null;
|
||||
},
|
||||
isActive: (id) => sessions.has(id),
|
||||
getState: (id) => {
|
||||
const session = sessions.get(id);
|
||||
if (!session) return null;
|
||||
const { worker, socket, closed, done, history, ...state } = session;
|
||||
return { ...state, startedBy: socket.data?.nickname || socket.id, socketId: socket.id };
|
||||
},
|
||||
shutdown: async () => {
|
||||
stopping = true;
|
||||
clearInterval(watchdog);
|
||||
unsubscribe?.();
|
||||
roverManager.managerEvents.off('sensor', sensor);
|
||||
await Promise.all([...sessions.keys()].map((id) => stop(id, 'Service disabled')));
|
||||
},
|
||||
kill: () => { for (const session of sessions.values()) session.worker.kill('SIGKILL'); },
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { createDriving, canControl };
|
||||
@@ -0,0 +1,39 @@
|
||||
// The only resident part while disabled is this configuration lifecycle bridge.
|
||||
const { loadConfig, registerConfigurationHandler } = require('../../configuration');
|
||||
const logger = require('../../globals/logger').child('roverLearning');
|
||||
|
||||
let runtime = null;
|
||||
let transitions = Promise.resolve();
|
||||
let shuttingDown = false;
|
||||
|
||||
function apply(config) {
|
||||
const transition = transitions.then(async () => {
|
||||
if (runtime) {
|
||||
const previous = runtime;
|
||||
runtime = null;
|
||||
await previous.stop();
|
||||
}
|
||||
if (!config?.enabled || shuttingDown) return;
|
||||
const { createRuntime } = require('./runtime');
|
||||
const next = createRuntime({ config, logger });
|
||||
try {
|
||||
await next.start();
|
||||
runtime = next;
|
||||
} catch (error) {
|
||||
await next.stop();
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
transitions = transition.catch(() => undefined);
|
||||
return transition;
|
||||
}
|
||||
|
||||
registerConfigurationHandler('roverLearning', apply);
|
||||
apply(loadConfig().roverLearning).catch((error) => {
|
||||
logger.error('Unable to start rover learning recording', { error: error.message });
|
||||
});
|
||||
|
||||
// Signal handlers are installed by the enabled runtime, not by this bridge.
|
||||
module.exports = {
|
||||
stop: () => { shuttingDown = true; return apply({ enabled: false }); },
|
||||
};
|
||||
@@ -0,0 +1,119 @@
|
||||
// Mutable aliases reference immutable checkpoints. All readers and retention share
|
||||
// a queue so a model cannot disappear between resolving its alias and loading it.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { randomUUID, randomInt } = require('node:crypto');
|
||||
const UUID = /^[0-9a-f-]{36}$/;
|
||||
let queue = Promise.resolve();
|
||||
const leases = new Map();
|
||||
const retired = new Set();
|
||||
function serial(task) {
|
||||
const result = queue.then(task);
|
||||
queue = result.catch(() => {});
|
||||
return result;
|
||||
}
|
||||
const checkpointRoot = (root) => path.join(root, 'training', 'checkpoints');
|
||||
async function index(root) {
|
||||
try { return JSON.parse(await fs.readFile(path.join(checkpointRoot(root), 'index.json'), 'utf8')); }
|
||||
catch (error) { if (error.code === 'ENOENT') return {}; throw error; }
|
||||
}
|
||||
async function metadata(directory) {
|
||||
const model = JSON.parse(await fs.readFile(path.join(directory, 'model.json'), 'utf8'));
|
||||
if (model.id !== path.basename(directory)) throw new Error('Model identity mismatch');
|
||||
return model;
|
||||
}
|
||||
async function prune(root, state) {
|
||||
const directory = checkpointRoot(root);
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (/^\.[0-9a-f-]{36}\.pending$/.test(entry.name)) {
|
||||
await fs.rm(path.join(directory, entry.name), { recursive: true });
|
||||
continue;
|
||||
}
|
||||
if (!UUID.test(entry.name)) continue;
|
||||
const target = path.join(directory, entry.name);
|
||||
if (!Object.values(state).includes(entry.name)) {
|
||||
if (leases.has(target)) retired.add(target);
|
||||
else { await fs.rm(target, { recursive: true }); retired.delete(target); }
|
||||
}
|
||||
}
|
||||
}
|
||||
function adoptCheckpoint(root, model, promote) {
|
||||
return serial(async () => {
|
||||
const state = await index(root);
|
||||
if (!UUID.test(model.id)) throw new Error('Invalid checkpoint identity');
|
||||
await metadata(path.join(checkpointRoot(root), model.id));
|
||||
state.candidate = model.id;
|
||||
if (promote) { state.previous = state.latest || null; state.latest = model.id; }
|
||||
const filename = path.join(checkpointRoot(root), 'index.json');
|
||||
await fs.writeFile(`${filename}.tmp`, JSON.stringify(state));
|
||||
await fs.rename(`${filename}.tmp`, filename);
|
||||
await prune(root, state);
|
||||
});
|
||||
}
|
||||
function acquireModel(root, id) {
|
||||
return serial(async () => {
|
||||
const state = await index(root);
|
||||
const alias = ['latest', 'previous'].includes(id);
|
||||
const resolved = alias ? state[id] : id;
|
||||
if (!UUID.test(resolved || '')) throw new Error('Model is not available');
|
||||
const directory = path.join(alias ? checkpointRoot(root) : path.join(root, 'models'), resolved);
|
||||
const model = await metadata(directory);
|
||||
leases.set(directory, (leases.get(directory) || 0) + 1);
|
||||
let released = false;
|
||||
return { directory, metadata: model, release: () => serial(async () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
const count = leases.get(directory) - 1;
|
||||
if (count) leases.set(directory, count); else leases.delete(directory);
|
||||
// Delete only a specific checkpoint already retired by the trainer, not
|
||||
// a worker's new export awaiting adoption.
|
||||
if (!count && retired.has(directory)) {
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
retired.delete(directory);
|
||||
}
|
||||
}) };
|
||||
});
|
||||
}
|
||||
function saveSnapshot(root) {
|
||||
return serial(async () => {
|
||||
const state = await index(root);
|
||||
if (!UUID.test(state.candidate || '')) throw new Error('No completed learner checkpoint yet');
|
||||
const source = path.join(checkpointRoot(root), state.candidate);
|
||||
const original = await metadata(source);
|
||||
const id = randomUUID();
|
||||
const words = [['amber', 'silver', 'quiet', 'curious'], ['otter', 'finch', 'fox', 'wren']];
|
||||
const name = `${words[0][randomInt(4)]}-${words[1][randomInt(4)]}-${id.slice(0, 8)}`;
|
||||
const model = { ...original, id, name, createdAt: Date.now(), sourceCheckpointId: original.id,
|
||||
snapshot: true, experimental: original.evaluation !== 'passed imitation gate' };
|
||||
const destination = path.join(root, 'models', id);
|
||||
const pending = path.join(root, 'models', `.${id}.pending`);
|
||||
await fs.mkdir(pending, { recursive: true });
|
||||
try {
|
||||
await fs.copyFile(path.join(source, 'weights.pt'), path.join(pending, 'weights.pt'));
|
||||
await fs.writeFile(path.join(pending, 'model.json'), JSON.stringify(model));
|
||||
await fs.rename(pending, destination);
|
||||
} catch (error) { await fs.rm(pending, { recursive: true, force: true }); throw error; }
|
||||
return model;
|
||||
});
|
||||
}
|
||||
function listModels(root) {
|
||||
return serial(async () => {
|
||||
let entries;
|
||||
try { entries = await fs.readdir(path.join(root, 'models'), { withFileTypes: true }); }
|
||||
catch (error) { if (error.code !== 'ENOENT') throw error; entries = []; }
|
||||
const models = [];
|
||||
const state = await index(root);
|
||||
for (const alias of ['latest', 'previous']) {
|
||||
if (!state[alias]) continue;
|
||||
const model = await metadata(path.join(checkpointRoot(root), state[alias]));
|
||||
models.push({ ...model, id: alias, checkpointId: model.id, name: alias === 'latest' ? 'Latest' : 'Previous', automatic: true });
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory() && UUID.test(entry.name)) models.push(await metadata(path.join(root, 'models', entry.name)));
|
||||
}
|
||||
return models;
|
||||
});
|
||||
}
|
||||
module.exports = { listModels, acquireModel, adoptCheckpoint, saveSnapshot };
|
||||
@@ -0,0 +1,182 @@
|
||||
// One bounded recording session. Video packets retain receiver wall-clock PTS;
|
||||
// telemetry uses the same server clock, not an assumed browser playback clock.
|
||||
const fs = require('node:fs');
|
||||
const fsp = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const { spawn } = require('node:child_process');
|
||||
const { controlProfile } = require('./capabilities');
|
||||
|
||||
const MAX_BUFFER_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
async function saveMetadata(directory, metadata) {
|
||||
const filename = path.join(directory, 'session.json');
|
||||
await fsp.writeFile(`${filename}.tmp`, JSON.stringify(metadata, null, 2));
|
||||
await fsp.rename(`${filename}.tmp`, filename);
|
||||
}
|
||||
|
||||
async function createRecording({ root, roverId, snapshot, logger, docked = false }) {
|
||||
const id = `${Date.now()}-${randomUUID()}`;
|
||||
const directory = path.join(root, id);
|
||||
await fsp.mkdir(directory);
|
||||
const metadata = {
|
||||
version: 1, id, roverId, startedAt: Date.now(), endedAt: null,
|
||||
controlProfile: controlProfile(snapshot.meta), wheelSpeedLimit: snapshot.meta?.maxWheelSpeed ?? 500,
|
||||
timing: 'Video PTS uses server receiver Unix seconds. Events use Unix milliseconds and process monotonic nanoseconds. Browser display delay is not measured.',
|
||||
video: { file: 'video.mkv', codec: 'source-copy', timestampUnit: 'seconds', origin: 'unix' },
|
||||
events: 'events.ndjson', droppedEvents: 0, bufferOnly: docked,
|
||||
trainingStartedAt: docked ? null : Date.now(),
|
||||
};
|
||||
await saveMetadata(directory, metadata);
|
||||
const output = fs.createWriteStream(path.join(directory, 'events.ndjson'), { flags: 'wx' });
|
||||
let buffered = [];
|
||||
let bufferedBytes = 0;
|
||||
let dockedAt = null;
|
||||
let pruning = Promise.resolve();
|
||||
let failure = null;
|
||||
let stopping = false;
|
||||
let stopPromise;
|
||||
let child;
|
||||
let childClosed = false;
|
||||
let stderr = '';
|
||||
let videoResult = null;
|
||||
let resolveVideo;
|
||||
const videoClosed = new Promise((resolve) => { resolveVideo = resolve; });
|
||||
output.on('error', (error) => {
|
||||
failure = error.message;
|
||||
// Stop file growth immediately when telemetry is no longer writable.
|
||||
child?.kill('SIGTERM');
|
||||
});
|
||||
|
||||
function append(kind, data, timestamp = {}) {
|
||||
if (stopping || failure) return false;
|
||||
try {
|
||||
const line = JSON.stringify({
|
||||
ts: Date.now(), monotonicNs: process.hrtime.bigint().toString(), ...timestamp, kind, data,
|
||||
}) + '\n';
|
||||
if (metadata.bufferOnly) {
|
||||
const now = Date.now();
|
||||
buffered.push({ at: now, line });
|
||||
bufferedBytes += Buffer.byteLength(line);
|
||||
while (buffered.length && buffered[0].at < now - 5000) {
|
||||
bufferedBytes -= Buffer.byteLength(buffered.shift().line);
|
||||
}
|
||||
while (bufferedBytes > MAX_BUFFER_BYTES && buffered.length) {
|
||||
metadata.droppedEvents += 1;
|
||||
bufferedBytes -= Buffer.byteLength(buffered.shift().line);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// Never wait for disk from a control/sensor callback. Explicit loss counts
|
||||
// prevent an overloaded recorder from pretending its history is complete.
|
||||
if (output.writableLength + Buffer.byteLength(line) > MAX_BUFFER_BYTES) {
|
||||
metadata.droppedEvents += 1;
|
||||
return true;
|
||||
}
|
||||
output.write(line);
|
||||
} catch (error) {
|
||||
metadata.droppedEvents += 1;
|
||||
failure = error.message;
|
||||
child?.kill('SIGTERM');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
append('snapshot', snapshot);
|
||||
// Read the local published stream, as replay does, without depending on the
|
||||
// replay service or re-encoding camera video. No audio/chat is captured.
|
||||
child = spawn(process.env.FFMPEG_BIN || 'ffmpeg', [
|
||||
'-hide_banner', '-loglevel', 'warning', '-nostdin', '-n',
|
||||
'-rtsp_transport', 'tcp', '-timeout', '10000000',
|
||||
'-use_wallclock_as_timestamps', '1', '-copyts',
|
||||
'-i', `rtsp://127.0.0.1:8554/${encodeURIComponent(roverId)}`,
|
||||
'-map', '0:v:0', '-an', '-c:v', 'copy', '-avoid_negative_ts', 'disabled',
|
||||
// Keyframe-aligned chunks keep a disposable dock buffer without repeatedly
|
||||
// reconnecting to RTSP. Extra edge packets are filtered by trainingStartedAt.
|
||||
'-f', 'segment', '-segment_time', '2', '-segment_format', 'matroska',
|
||||
'-reset_timestamps', '0', path.join(directory, 'video-%09d.mkv'),
|
||||
], { stdio: ['ignore', 'ignore', 'pipe'] });
|
||||
child.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString()).slice(-4096); });
|
||||
child.on('error', (error) => { failure = error.message; });
|
||||
child.once('close', (code, signal) => {
|
||||
childClosed = true;
|
||||
videoResult = { code, signal, stderr };
|
||||
if (!stopping) failure ||= `Video recorder exited (${code ?? signal})`;
|
||||
resolveVideo();
|
||||
});
|
||||
|
||||
function updateDock(isDocked) {
|
||||
if (stopping) return;
|
||||
if (!isDocked) {
|
||||
dockedAt = null;
|
||||
if (metadata.bufferOnly) {
|
||||
metadata.bufferOnly = false;
|
||||
metadata.trainingStartedAt = Date.now() - 5000;
|
||||
for (const item of buffered) output.write(item.line);
|
||||
buffered = [];
|
||||
bufferedBytes = 0;
|
||||
}
|
||||
} else if (!metadata.bufferOnly && dockedAt == null) dockedAt = Date.now();
|
||||
}
|
||||
|
||||
function pruneBuffer() {
|
||||
pruning = pruning.then(async () => {
|
||||
if (!metadata.bufferOnly || stopping) return;
|
||||
const files = (await fsp.readdir(directory)).filter((name) => /^video-\d+\.mkv$/.test(name)).sort();
|
||||
// Never unlink the open segment; retain five seconds plus the preceding
|
||||
// keyframe segment so the decoder can recover the full lead-in.
|
||||
for (const name of files.slice(0, -2)) {
|
||||
const filename = path.join(directory, name);
|
||||
const stat = await fsp.stat(filename);
|
||||
if (!metadata.bufferOnly || stopping) return;
|
||||
if (stat.mtimeMs < Date.now() - 5000) await fsp.unlink(filename);
|
||||
}
|
||||
}).catch((error) => { failure = error.message; child.kill('SIGTERM'); });
|
||||
}
|
||||
|
||||
async function stop(reason) {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
append('sessionEnd', { reason, droppedEvents: metadata.droppedEvents });
|
||||
stopping = true;
|
||||
if (!childClosed) child.kill('SIGTERM');
|
||||
// A stuck RTSP read must not leave a recorder behind after disable.
|
||||
const killTimer = setTimeout(() => { if (!childClosed) child.kill('SIGKILL'); }, 2000);
|
||||
await videoClosed;
|
||||
await pruning;
|
||||
clearTimeout(killTimer);
|
||||
await new Promise((resolve) => {
|
||||
if (output.closed) return resolve();
|
||||
output.once('close', resolve);
|
||||
output.end();
|
||||
});
|
||||
if (metadata.bufferOnly) {
|
||||
await fsp.rm(directory, { recursive: true, force: true });
|
||||
return;
|
||||
}
|
||||
metadata.video.files = (await fsp.readdir(directory)).filter((name) => /^video-\d+\.mkv$/.test(name)).sort();
|
||||
metadata.endedAt = Date.now();
|
||||
metadata.reason = reason;
|
||||
metadata.error = failure;
|
||||
metadata.video.result = videoResult;
|
||||
await saveMetadata(directory, metadata);
|
||||
if (failure) logger.warn('Recording session incomplete', { roverId, id, error: failure, stderr });
|
||||
})();
|
||||
return stopPromise;
|
||||
}
|
||||
|
||||
return {
|
||||
id, directory, startedAt: metadata.startedAt, append, stop, updateDock, pruneBuffer,
|
||||
get buffered() { return metadata.bufferOnly; },
|
||||
get dockTailComplete() { return dockedAt != null && Date.now() - dockedAt >= 5000; },
|
||||
get trainingStartedAt() { return metadata.trainingStartedAt; },
|
||||
get failed() { return Boolean(failure); },
|
||||
get stopping() { return stopping; },
|
||||
getStatus: () => ({ id, roverId, startedAt: metadata.startedAt, droppedEvents: metadata.droppedEvents,
|
||||
video: childClosed ? 'stopped' : metadata.bufferOnly ? 'Dock lead-in buffer' : 'Recording human driving', error: failure, stopping }),
|
||||
// Process exit is synchronous; normal disable uses stop() and awaits close.
|
||||
kill: () => { if (!childClosed) child.kill('SIGKILL'); },
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { createRecording, saveMetadata };
|
||||
@@ -0,0 +1,311 @@
|
||||
// Enabled-only recording, training, controller, and card lifecycle.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { resolveDataPath } = require('../../helpers/dataPaths');
|
||||
const roverManager = require('../roverManager');
|
||||
const io = require('../../globals/io');
|
||||
const { turnEvents } = require('../turnService');
|
||||
const { modeEvents } = require('../modeManager');
|
||||
const { isAdmin, roleEvents } = require('../roleService');
|
||||
const { isDeterred, verificationEvents } = require('../verificationService');
|
||||
const { subscribeCommandRecording, commandEvents } = require('../commandService');
|
||||
const { createRecording } = require('./recording');
|
||||
const { recoverRecordings, enforceRetention } = require('./storage');
|
||||
const { createTrainer } = require('./training');
|
||||
const { createDriving } = require('./driving');
|
||||
const { createGateway } = require('./socketGateway');
|
||||
|
||||
function createRuntime({ config, logger }) {
|
||||
const root = resolveDataPath('rover-learning');
|
||||
const recordingsRoot = path.join(root, 'recordings');
|
||||
const recentActivity = [];
|
||||
function activity(message, details = {}) {
|
||||
recentActivity.push({ at: Date.now(), message, roverId: details.roverId || null,
|
||||
detail: details.reason || details.error || details.model || null });
|
||||
if (recentActivity.length > 40) recentActivity.shift();
|
||||
}
|
||||
const baseLogger = logger;
|
||||
logger = Object.fromEntries(['info', 'warn', 'error'].map((level) => [level, (message, details) => {
|
||||
baseLogger[level](message, details); activity(message, details);
|
||||
}]));
|
||||
const trainer = createTrainer({ root, config: config.training, actionThreshold: config.driving.actionThreshold, logger });
|
||||
const driving = createDriving({ root, config, onChange: refreshEligibility, activity });
|
||||
let gateway;
|
||||
let dockTimer;
|
||||
let diskState = { bytes: null, free: null };
|
||||
const sessions = new Map();
|
||||
const unsubscribers = [];
|
||||
let running = false;
|
||||
let timer;
|
||||
let maintenance = Promise.resolve();
|
||||
let stopPromise;
|
||||
let diskPaused = false;
|
||||
let busy = false;
|
||||
let wakeRequested = false;
|
||||
let pending = [];
|
||||
let pendingBytes = 0;
|
||||
let droppedPending = 0;
|
||||
let eligibleRovers = new Set();
|
||||
|
||||
function hasHumanController(roverId) {
|
||||
// Even manual inputs during autonomy remain excluded: model-influenced
|
||||
// video must not silently become human demonstration training data.
|
||||
if (driving.isActive(roverId)) return false;
|
||||
const record = roverManager.rovers.get(roverId);
|
||||
if (!record?.ws) return false;
|
||||
// Admin permission alone must not record the whole fleet. Require actual
|
||||
// driver membership, a live browser socket, and current control eligibility.
|
||||
for (const socketId of record.drivers || []) {
|
||||
const socket = io.sockets.sockets.get(socketId);
|
||||
if (socket?.connected && (isAdmin(socket) || !isDeterred(socket))
|
||||
&& roverManager.canDrive(roverId, socket)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function currentEligibleRovers() {
|
||||
return new Set([...roverManager.rovers.keys()].filter(hasHumanController));
|
||||
}
|
||||
|
||||
function refreshEligibility() {
|
||||
const next = currentEligibleRovers();
|
||||
const changed = next.size !== eligibleRovers.size || [...next].some((id) => !eligibleRovers.has(id));
|
||||
eligibleRovers = next;
|
||||
if (!changed) return;
|
||||
// Stop the stream immediately even if disk maintenance is in progress.
|
||||
// The serialized maintenance pass still owns closing/removing the session.
|
||||
for (const [id, session] of sessions) {
|
||||
if (!next.has(id)) session.stop('no-human-controller').catch((error) => {
|
||||
logger.warn('Unable to finalize unattended recording', { roverId: id, error: error.message });
|
||||
});
|
||||
}
|
||||
schedule();
|
||||
}
|
||||
|
||||
function append(kind, event) {
|
||||
if (!running || event?.roverId == null) return;
|
||||
const roverId = String(event.roverId);
|
||||
if (!hasHumanController(roverId)) return;
|
||||
const timestamp = { ts: Date.now(), monotonicNs: process.hrtime.bigint().toString() };
|
||||
if (sessions.get(roverId)?.append(kind, event, timestamp)) return;
|
||||
if (diskPaused || !roverManager.rovers.has(roverId)) return;
|
||||
// Session creation/rotation is asynchronous. Keep a bounded snapshot of
|
||||
// events arriving during that gap, with their original observation times.
|
||||
try {
|
||||
const serialized = JSON.stringify(event);
|
||||
const bytes = Buffer.byteLength(serialized);
|
||||
if (pendingBytes + bytes > 4 * 1024 * 1024) {
|
||||
droppedPending += 1;
|
||||
return;
|
||||
}
|
||||
pending.push({ roverId, kind, serialized, timestamp, bytes });
|
||||
pendingBytes += bytes;
|
||||
} catch { droppedPending += 1; }
|
||||
}
|
||||
|
||||
function listen(emitter, name, handler) {
|
||||
emitter.on(name, handler);
|
||||
unsubscribers.push(() => emitter.off(name, handler));
|
||||
}
|
||||
|
||||
function snapshot(record) {
|
||||
// Do not serialize the live record: it contains sockets and circular state.
|
||||
return {
|
||||
meta: record.meta, sensors: record.lastSensor, hostStats: record.lastHostStats,
|
||||
batteryState: record.batteryState, headlightState: record.headlightState,
|
||||
laserState: record.laserState, drivers: [...(record.drivers || [])],
|
||||
locked: record.locked, privateOpen: record.privateOpen,
|
||||
};
|
||||
}
|
||||
|
||||
async function closeSession(roverId, reason) {
|
||||
const session = sessions.get(roverId);
|
||||
if (!session) return;
|
||||
try {
|
||||
await session.stop(reason);
|
||||
} finally {
|
||||
// Keep closing children reachable by the synchronous process-exit hook.
|
||||
sessions.delete(roverId);
|
||||
}
|
||||
}
|
||||
|
||||
async function maintain() {
|
||||
if (!running) return;
|
||||
pending = pending.filter((event) => hasHumanController(event.roverId));
|
||||
pendingBytes = pending.reduce((sum, event) => sum + event.bytes, 0);
|
||||
for (const [roverId, session] of sessions) {
|
||||
const record = roverManager.rovers.get(roverId);
|
||||
if (!hasHumanController(roverId) || session.stopping || session.failed || session.dockTailComplete || (!session.buffered && Date.now() - session.trainingStartedAt >= config.recording.sessionSeconds * 1000)) {
|
||||
await closeSession(roverId, !record?.ws ? 'offline' : !hasHumanController(roverId) ? 'no-human-controller' : session.failed ? 'recorder-failed' : session.dockTailComplete ? 'docked' : 'rotation');
|
||||
}
|
||||
}
|
||||
if (!running) return;
|
||||
const retention = await enforceRetention(
|
||||
recordingsRoot, new Set([...sessions.values()].map((session) => session.id).concat([...trainer.leases])),
|
||||
config.recording.maxGiB * 1024 ** 3, config.recording.minimumFreeGiB * 1024 ** 3,
|
||||
);
|
||||
diskState = retention;
|
||||
if (retention.paused) {
|
||||
if (!diskPaused) logger.warn('Recording paused for disk budget', retention);
|
||||
diskPaused = true;
|
||||
pending = [];
|
||||
pendingBytes = 0;
|
||||
await trainer.cancel();
|
||||
// Close active files to make them eligible for pruning on the next tick.
|
||||
await Promise.all([...sessions.keys()].map((id) => closeSession(id, 'disk-budget')));
|
||||
return;
|
||||
}
|
||||
if (diskPaused) logger.info('Recording resumed after disk budget recovery');
|
||||
diskPaused = false;
|
||||
for (const [key, record] of roverManager.rovers) {
|
||||
if (!running) break;
|
||||
const roverId = String(key);
|
||||
if (!hasHumanController(roverId) || sessions.has(roverId)) continue;
|
||||
const session = await createRecording({ root: recordingsRoot, roverId, snapshot: snapshot(record), logger, docked: Boolean(record.lastSensor?.decoded?.chargingSources?.homeBase) });
|
||||
sessions.set(roverId, session);
|
||||
// Filesystem awaits can span a disconnect or turn change.
|
||||
if (!running || !hasHumanController(roverId)) {
|
||||
await closeSession(roverId, running ? 'no-human-controller' : 'disabled');
|
||||
continue;
|
||||
}
|
||||
if (droppedPending) {
|
||||
session.append('captureGap', { droppedEventsAcrossRovers: droppedPending });
|
||||
droppedPending = 0;
|
||||
}
|
||||
pending = pending.filter((event) => {
|
||||
if (event.roverId !== roverId) return true;
|
||||
session.append(event.kind, JSON.parse(event.serialized), event.timestamp);
|
||||
pendingBytes -= event.bytes;
|
||||
return false;
|
||||
});
|
||||
}
|
||||
if (running) {
|
||||
try {
|
||||
await trainer.tick(new Set([...sessions.values()].map((session) => session.id)));
|
||||
} catch (error) {
|
||||
// Training availability must not tear down otherwise healthy capture.
|
||||
logger.error('Unable to schedule training', { error: error.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
if (!running) return;
|
||||
if (busy) { wakeRequested = true; return; }
|
||||
clearTimeout(timer);
|
||||
busy = true;
|
||||
// The next tick is scheduled after completion: disk slowness cannot build
|
||||
// an unbounded backlog of retention tasks or overlapping recorder starts.
|
||||
maintenance = maintain().catch(async (error) => {
|
||||
logger.error('Recording maintenance failed; closing recorders', { error: error.message });
|
||||
await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'storage-error')));
|
||||
}).finally(() => {
|
||||
busy = false;
|
||||
if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
|
||||
wakeRequested = false;
|
||||
});
|
||||
}
|
||||
|
||||
function killChildren() {
|
||||
driving.kill();
|
||||
trainer.kill();
|
||||
for (const session of sessions.values()) session.kill();
|
||||
}
|
||||
function onSignal() {
|
||||
// Other server services can exit the process before asynchronous cleanup
|
||||
// finishes. Signal children immediately, and keep an exit fallback too.
|
||||
for (const session of sessions.values()) session.stop('server-shutdown').catch(() => undefined);
|
||||
stop().catch((error) => logger.warn('Recording shutdown failed', { error: error.message }));
|
||||
}
|
||||
|
||||
async function start() {
|
||||
await fs.mkdir(recordingsRoot, { recursive: true });
|
||||
await recoverRecordings(recordingsRoot);
|
||||
await trainer.start();
|
||||
running = true;
|
||||
driving.attach();
|
||||
dockTimer = setInterval(() => {
|
||||
for (const [id, session] of sessions) {
|
||||
session.updateDock(Boolean(roverManager.rovers.get(id)?.lastSensor?.decoded?.chargingSources?.homeBase));
|
||||
session.pruneBuffer();
|
||||
if (session.dockTailComplete) schedule();
|
||||
}
|
||||
}, 250);
|
||||
gateway = createGateway({ root, driving, activity, getSystemState: () => ({
|
||||
recording: { paused: diskPaused, bytes: diskState.bytes, freeBytes: diskState.free,
|
||||
pendingBytes, droppedPending, sessions: [...sessions.values()].map((session) => session.getStatus()) },
|
||||
training: trainer.getState(),
|
||||
limits: { recording: config.recording, training: { ...config.training, python: undefined }, driving: config.driving },
|
||||
activity: [...recentActivity].reverse(),
|
||||
}) });
|
||||
eligibleRovers = currentEligibleRovers();
|
||||
unsubscribers.push(subscribeCommandRecording((event) => append('command', event)));
|
||||
// ACK analytics also provide latency/type context; detailed acknowledgements
|
||||
// arrive through the recording hook and share the command ID.
|
||||
listen(commandEvents, 'observation', (event) => append('commandOutcome', event));
|
||||
listen(roverManager.managerEvents, 'sensor', (event) => append('sensor', {
|
||||
...event,
|
||||
// The manager stores the exact frame before emitting this synchronous
|
||||
// event, so raw capture needs no extra hook in rover sensor processing.
|
||||
frame: roverManager.rovers.get(event.roverId)?.lastSensor?.raw,
|
||||
}));
|
||||
for (const kind of ['hostStats', 'driver', 'switch', 'lock', 'private', 'privateSafety', 'help', 'dockGuard']) {
|
||||
listen(roverManager.managerEvents, kind, (event) => {
|
||||
append(kind, event);
|
||||
if (['driver', 'switch', 'lock', 'private', 'privateSafety'].includes(kind)) refreshEligibility();
|
||||
});
|
||||
}
|
||||
for (const kind of ['activeDriver', 'queue']) {
|
||||
listen(turnEvents, kind, (event) => { append(`turn:${kind}`, event); refreshEligibility(); });
|
||||
}
|
||||
listen(modeEvents, 'change', refreshEligibility);
|
||||
listen(roleEvents, 'change', refreshEligibility);
|
||||
listen(verificationEvents, 'change', refreshEligibility);
|
||||
listen(roverManager.managerEvents, 'rover', (event) => {
|
||||
append('rover', { roverId: event.roverId, action: event.action, snapshot: event.record ? snapshot(event.record) : null });
|
||||
if (event.action === 'upsert' || event.action === 'removed') refreshEligibility();
|
||||
});
|
||||
process.on('SIGTERM', onSignal);
|
||||
process.on('SIGINT', onSignal);
|
||||
process.on('exit', killChildren);
|
||||
busy = true;
|
||||
maintenance = maintain();
|
||||
try {
|
||||
await maintenance;
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
if (running) timer = setTimeout(schedule, wakeRequested ? 0 : 15000);
|
||||
wakeRequested = false;
|
||||
logger.info('Rover learning enabled', { root, training: config.training.enabled });
|
||||
}
|
||||
|
||||
async function stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
running = false;
|
||||
gateway?.close();
|
||||
clearTimeout(timer);
|
||||
clearInterval(dockTimer);
|
||||
for (const unsubscribe of unsubscribers.splice(0)) unsubscribe();
|
||||
stopPromise = (async () => {
|
||||
const trainingStop = trainer.stop();
|
||||
const drivingStop = driving.shutdown();
|
||||
await maintenance.catch(() => undefined);
|
||||
const results = await Promise.allSettled([...sessions.keys()].map((id) => closeSession(id, 'disabled')));
|
||||
await trainingStop;
|
||||
await drivingStop;
|
||||
process.off('SIGTERM', onSignal);
|
||||
process.off('SIGINT', onSignal);
|
||||
process.off('exit', killChildren);
|
||||
pending = [];
|
||||
pendingBytes = 0;
|
||||
const failed = results.find((result) => result.status === 'rejected');
|
||||
if (failed) throw failed.reason;
|
||||
})();
|
||||
return stopPromise;
|
||||
}
|
||||
|
||||
return { start, stop, getTrainingState: trainer.getState };
|
||||
}
|
||||
|
||||
module.exports = { createRuntime };
|
||||
@@ -0,0 +1,127 @@
|
||||
// Complete card snapshots; live diagnostics opt into 10 Hz, ordinary cards use 1 Hz.
|
||||
const io = require('../../globals/io');
|
||||
const roverManager = require('../roverManager');
|
||||
const { isVerified } = require('../verificationService');
|
||||
const { isAdmin } = require('../roleService');
|
||||
const { listModels, saveSnapshot } = require('./models');
|
||||
const { report } = require('./diagnostics');
|
||||
const { compatible } = require('./capabilities');
|
||||
|
||||
function createGateway({ root, driving, getSystemState, activity }) {
|
||||
const bindings = new Map();
|
||||
const viewers = new Map();
|
||||
let models = [];
|
||||
let modelError = null;
|
||||
let lastCatalog = 0;
|
||||
let closed = false;
|
||||
let refreshing = false;
|
||||
let lastOrdinary = 0;
|
||||
|
||||
function stateFor(socket, roverId) {
|
||||
if (!isVerified(socket)) return { available: false };
|
||||
const visible = roverId && roverManager.canSeeRover(roverId, socket);
|
||||
const id = visible ? roverId : null;
|
||||
const session = id ? driving.getState(id) : null;
|
||||
const canControl = Boolean(id && driving.canControl(socket, id));
|
||||
const system = getSystemState();
|
||||
const state = {
|
||||
available: true, roverId: id, updatedAt: Date.now(),
|
||||
control: { canSnapshot: isAdmin(socket), canStart: canControl && !session, canAdjust: canControl && Boolean(session), canStop: Boolean(session && (canControl || session.socketId === socket.id)),
|
||||
reason: !id ? 'Select a rover first' : !canControl ? 'You do not currently have control of this rover' : null },
|
||||
session,
|
||||
models: models.filter((model) => model.trainedRovers?.some((rover) => roverManager.canSeeRover(rover, socket)))
|
||||
.map((model) => ({ id: model.id, checkpointId: model.checkpointId || model.id, automatic: Boolean(model.automatic), experimental: Boolean(model.experimental), name: model.name, createdAt: model.createdAt,
|
||||
trainingSteps: model.trainingSteps, examplesProcessed: model.examplesProcessed,
|
||||
weightBytes: model.weightBytes, parameters: model.parameters, trainingLoss: model.trainingLoss,
|
||||
formatVersion: model.specification?.version, evaluation: model.evaluation, validation: model.validation, distinctMinutes: model.distinctMinutes, controls: model.specification?.commands?.length || 0,
|
||||
compatible: Boolean(id && compatible(model, roverManager.rovers.get(id))) })),
|
||||
catalogError: modelError,
|
||||
recording: { ...system.recording, sessions: system.recording.sessions.filter((s) => roverManager.canSeeRover(s.roverId, socket)) },
|
||||
training: { ...system.training, sessions: undefined },
|
||||
limits: system.limits,
|
||||
activity: system.activity.filter((entry) => !entry.roverId || roverManager.canSeeRover(entry.roverId, socket)),
|
||||
};
|
||||
state.diagnostics = viewers.get(socket)?.live ? report(state, id ? driving.getDiagnostics(id) : null) : null;
|
||||
return state;
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
if (closed || refreshing) return;
|
||||
refreshing = true;
|
||||
try {
|
||||
if (Date.now() - lastCatalog >= 15000) {
|
||||
lastCatalog = Date.now();
|
||||
try { models = await listModels(root); modelError = null; }
|
||||
catch (error) { modelError = error.message; }
|
||||
}
|
||||
if (closed) return;
|
||||
const ordinary = Date.now() - lastOrdinary >= 1000;
|
||||
if (ordinary) lastOrdinary = Date.now();
|
||||
for (const [socket, view] of viewers) {
|
||||
if (ordinary || view.live || driving.isActive(view.roverId)) socket.volatile.emit('roverLearning:state', stateFor(socket, view.roverId));
|
||||
}
|
||||
} finally { refreshing = false; }
|
||||
}
|
||||
|
||||
function connect(socket) {
|
||||
const subscribe = (payload = {}, reply) => {
|
||||
const respond = typeof reply === 'function' ? reply : () => {};
|
||||
if (!isVerified(socket)) { respond({ error: 'Verification required' }); return; }
|
||||
const roverId = typeof payload?.roverId === 'string' ? payload.roverId : null;
|
||||
viewers.set(socket, { roverId, live: payload.live === true });
|
||||
respond({ state: stateFor(socket, roverId) });
|
||||
refresh().catch((error) => activity('Card update failed', { error: error.message }));
|
||||
};
|
||||
const action = async (payload = {}, reply) => {
|
||||
const respond = typeof reply === 'function' ? reply : () => {};
|
||||
try {
|
||||
if (!isVerified(socket)) throw new Error('Verification required');
|
||||
const roverId = String(payload.roverId || '');
|
||||
if (payload.action === 'snapshot') {
|
||||
if (!isAdmin(socket)) throw new Error('Administrator access required');
|
||||
const model = await saveSnapshot(root);
|
||||
models = await listModels(root);
|
||||
lastCatalog = Date.now();
|
||||
activity('Learner snapshot saved', { model: model.name });
|
||||
} else if (payload.action === 'start') await driving.start(socket, roverId, String(payload.modelId || ''));
|
||||
else if (payload.action === 'threshold') driving.setThreshold(socket, roverId, payload.value);
|
||||
else if (payload.action === 'stop') {
|
||||
const session = driving.getState(roverId);
|
||||
if (!session || !(driving.canControl(socket, roverId) || session.socketId === socket.id)) throw new Error('You cannot stop this controller');
|
||||
await driving.stop(roverId);
|
||||
} else throw new Error('Unknown action');
|
||||
viewers.set(socket, { roverId, live: viewers.get(socket)?.live || false });
|
||||
respond({ state: stateFor(socket, roverId) });
|
||||
} catch (error) { respond({ error: error.message }); }
|
||||
refresh().catch((error) => activity('Card update failed', { error: error.message }));
|
||||
};
|
||||
const unsubscribe = () => viewers.delete(socket);
|
||||
const disconnect = () => { unsubscribe(); detach(socket); };
|
||||
const handlers = { 'roverLearning:subscribe': subscribe, 'roverLearning:action': action,
|
||||
'roverLearning:unsubscribe': unsubscribe, disconnect };
|
||||
for (const [name, handler] of Object.entries(handlers)) socket.on(name, handler);
|
||||
bindings.set(socket, handlers);
|
||||
}
|
||||
|
||||
function detach(socket) {
|
||||
for (const [name, handler] of Object.entries(bindings.get(socket) || {})) socket.off(name, handler);
|
||||
bindings.delete(socket);
|
||||
}
|
||||
|
||||
io.on('connection', connect);
|
||||
for (const socket of io.sockets.sockets.values()) connect(socket);
|
||||
const timer = setInterval(() => { refresh().catch((error) => activity('Card update failed', { error: error.message })); }, 100);
|
||||
refresh().catch((error) => activity('Catalog unavailable', { error: error.message }));
|
||||
return {
|
||||
close() {
|
||||
closed = true;
|
||||
clearInterval(timer);
|
||||
io.off('connection', connect);
|
||||
for (const socket of viewers.keys()) socket.emit('roverLearning:state', { available: false });
|
||||
viewers.clear();
|
||||
for (const socket of bindings.keys()) detach(socket);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { createGateway };
|
||||
@@ -0,0 +1,63 @@
|
||||
// Recording retention only. Published models and training state are never visited.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { saveMetadata } = require('./recording');
|
||||
|
||||
async function listRecordings(root) {
|
||||
const recordings = [];
|
||||
for (const entry of await fs.readdir(root, { withFileTypes: true })) {
|
||||
// Directory names are generated, never rover IDs or user-supplied paths.
|
||||
if (!entry.isDirectory() || !/^\d+-[0-9a-f-]{36}$/.test(entry.name)) continue;
|
||||
const directory = path.join(root, entry.name);
|
||||
let bytes = 0;
|
||||
for (const file of await fs.readdir(directory, { withFileTypes: true })) {
|
||||
if (!file.isFile()) continue;
|
||||
bytes += (await fs.stat(path.join(directory, file.name))).size;
|
||||
}
|
||||
recordings.push({ id: entry.name, directory, bytes, startedAt: Number(entry.name.split('-')[0]) });
|
||||
}
|
||||
return recordings.sort((a, b) => a.startedAt - b.startedAt);
|
||||
}
|
||||
|
||||
async function recoverRecordings(root) {
|
||||
for (const recording of await listRecordings(root)) {
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8'));
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
|
||||
// An interrupted initial metadata write is retained as an unusable session
|
||||
// until ordinary retention removes it, rather than blocking all startup.
|
||||
metadata = { version: 1, id: recording.id, startedAt: recording.startedAt };
|
||||
}
|
||||
if (metadata.bufferOnly) {
|
||||
await fs.rm(recording.directory, { recursive: true, force: true });
|
||||
continue;
|
||||
}
|
||||
if (metadata.endedAt != null) continue;
|
||||
await saveMetadata(recording.directory, {
|
||||
...metadata, endedAt: Date.now(), reason: 'interrupted',
|
||||
error: 'Previous process did not close this session; final timestamps and video may be incomplete.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function enforceRetention(root, activeIds, maxBytes, minimumFreeBytes) {
|
||||
const recordings = await listRecordings(root);
|
||||
let bytes = recordings.reduce((total, recording) => total + recording.bytes, 0);
|
||||
const stat = await fs.statfs(root);
|
||||
let free = stat.bavail * stat.bsize;
|
||||
for (const recording of recordings) {
|
||||
if (bytes < maxBytes && free >= minimumFreeBytes) break;
|
||||
if (activeIds.has(recording.id)) continue;
|
||||
await fs.rm(recording.directory, { recursive: true });
|
||||
bytes -= recording.bytes;
|
||||
// Re-read actual free space; sparse/compressed files and other services can
|
||||
// make a deleted file's logical size a misleading estimate of freed space.
|
||||
const current = await fs.statfs(root);
|
||||
free = current.bavail * current.bsize;
|
||||
}
|
||||
return { bytes, free, paused: bytes >= maxBytes || free < minimumFreeBytes };
|
||||
}
|
||||
|
||||
module.exports = { recoverRecordings, enforceRetention, listRecordings };
|
||||
@@ -0,0 +1,185 @@
|
||||
// One owned finite worker at a time. Leases are acquired in the runtime's
|
||||
// serialized maintenance pass, before retention can next inspect recordings.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
const { randomInt } = require('node:crypto');
|
||||
const { adoptCheckpoint } = require('./models');
|
||||
const { listRecordings } = require('./storage');
|
||||
|
||||
function createTrainer({ root, config, actionThreshold, logger }) {
|
||||
const leases = new Set();
|
||||
const trainingRoot = path.join(root, 'training');
|
||||
let ledger = {};
|
||||
const history = [];
|
||||
let child = null;
|
||||
let completion = Promise.resolve();
|
||||
let stopped = false;
|
||||
let nextRun = 0;
|
||||
let startupError = false;
|
||||
let state = { status: config.enabled ? 'waiting' : 'disabled', steps: 0, examples: 0 };
|
||||
|
||||
async function start() {
|
||||
if (!config.enabled) return;
|
||||
try {
|
||||
await fs.mkdir(trainingRoot, { recursive: true });
|
||||
await fs.mkdir(path.join(root, 'models'), { recursive: true });
|
||||
// Only unpublished staging directories from an interrupted old worker
|
||||
// are disposable. Permanent UUID model directories are never removed.
|
||||
for (const entry of await fs.readdir(path.join(root, 'models'), { withFileTypes: true })) {
|
||||
if (entry.isDirectory() && /^\.[0-9a-f-]{36}\.pending$/.test(entry.name)) {
|
||||
await fs.rm(path.join(root, 'models', entry.name), { recursive: true });
|
||||
}
|
||||
}
|
||||
try {
|
||||
const saved = JSON.parse(await fs.readFile(path.join(trainingRoot, 'schedule-v3.json'), 'utf8'));
|
||||
ledger = saved.sessions;
|
||||
state = { ...saved.progress, status: 'waiting' };
|
||||
if (!ledger || typeof ledger !== 'object') throw new Error('Invalid training schedule');
|
||||
} catch (error) { if (error.code !== 'ENOENT') throw error; }
|
||||
} catch (error) {
|
||||
startupError = true;
|
||||
state = { ...state, status: 'error', error: error.message };
|
||||
logger.error('Training initialization failed; recording remains available', { error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async function saveLedger() {
|
||||
const filename = path.join(trainingRoot, 'schedule-v3.json');
|
||||
await fs.writeFile(`${filename}.tmp`, JSON.stringify({ sessions: ledger, progress: state }));
|
||||
await fs.rename(`${filename}.tmp`, filename);
|
||||
}
|
||||
|
||||
async function tick(activeIds) {
|
||||
if (!config.enabled || startupError || stopped || child || Date.now() < nextRun) return;
|
||||
nextRun = Date.now() + config.intervalSeconds * 1000;
|
||||
const recordings = await listRecordings(path.join(root, 'recordings'));
|
||||
const present = new Set(recordings.map((recording) => recording.id));
|
||||
// Scheduling history is bounded by recording retention, unlike models.
|
||||
for (const id of Object.keys(ledger)) if (!present.has(id)) delete ledger[id];
|
||||
const eligible = [];
|
||||
for (const recording of recordings) {
|
||||
if (activeIds.has(recording.id) || ledger[recording.id]?.rejected) continue;
|
||||
try {
|
||||
const metadata = JSON.parse(await fs.readFile(path.join(recording.directory, 'session.json'), 'utf8'));
|
||||
if (metadata.bufferOnly || !metadata.endedAt || metadata.error || metadata.droppedEvents || metadata.reason === 'interrupted') continue;
|
||||
eligible.push(recording.id);
|
||||
} catch (error) {
|
||||
ledger[recording.id] = { rejected: `Unreadable session metadata: ${error.message}` };
|
||||
}
|
||||
}
|
||||
// Random cohorts mix old and new retained sessions without repeatedly
|
||||
// favoring the first files in directory order.
|
||||
for (let index = eligible.length - 1; index > 0; index -= 1) {
|
||||
const other = randomInt(index + 1);
|
||||
[eligible[index], eligible[other]] = [eligible[other], eligible[index]];
|
||||
}
|
||||
const selected = eligible.slice(0, 4);
|
||||
nextRun = Date.now() + config.intervalSeconds * 1000;
|
||||
if (!selected.length) {
|
||||
state = { ...state, status: 'waiting', reason: 'Waiting for completed usable human recordings' };
|
||||
await saveLedger();
|
||||
return;
|
||||
}
|
||||
const jobPath = path.join(trainingRoot, 'job.json');
|
||||
await fs.writeFile(jobPath, JSON.stringify({ root, config, actionThreshold, sessions: selected }));
|
||||
if (stopped) return;
|
||||
selected.forEach((id) => leases.add(id));
|
||||
state = { ...state, status: 'training', reason: null, sessions: selected };
|
||||
state.startedAt = Date.now();
|
||||
child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1', OMP_NUM_THREADS: String(config.threads), MKL_NUM_THREADS: String(config.threads) },
|
||||
});
|
||||
let output = '';
|
||||
let stderr = '';
|
||||
let result = null;
|
||||
let errorMessage = null;
|
||||
let forcedTimer;
|
||||
const worker = child;
|
||||
const deadline = setTimeout(() => {
|
||||
worker.kill('SIGTERM');
|
||||
forcedTimer = setTimeout(() => worker.kill('SIGKILL'), 5000);
|
||||
}, (config.maxJobSeconds + 15) * 1000);
|
||||
child.on('error', (error) => { errorMessage = error.message; });
|
||||
child.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-8192); });
|
||||
child.stdout.on('data', (chunk) => {
|
||||
output += chunk;
|
||||
if (output.length > 1024 * 1024) {
|
||||
errorMessage = 'Training worker output exceeded protocol bound';
|
||||
worker.kill('SIGKILL');
|
||||
return;
|
||||
}
|
||||
let newline;
|
||||
while ((newline = output.indexOf('\n')) >= 0) {
|
||||
const line = output.slice(0, newline);
|
||||
output = output.slice(newline + 1);
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
if (message.kind === 'complete') result = message;
|
||||
if (message.kind === 'error') errorMessage = message.message;
|
||||
if (message.kind === 'skipped') logger.warn('Training skipped recording', message);
|
||||
if (message.kind === 'dataset') state = { ...state, lastSampledWindows: message.sampled, lastAvailableWindows: message.windows, dataset: message.diagnostics };
|
||||
if (message.kind === 'progress') state = { ...state, steps: message.steps, examples: message.examples, loss: message.loss, losses: message.losses };
|
||||
} catch { errorMessage = 'Invalid training worker message'; }
|
||||
}
|
||||
});
|
||||
completion = new Promise((resolve) => {
|
||||
child.once('close', async (code, signal) => {
|
||||
clearTimeout(deadline);
|
||||
clearTimeout(forcedTimer);
|
||||
try {
|
||||
if (code !== 0 || errorMessage || !result) {
|
||||
state = { ...state, status: stopped ? 'stopped' : 'error', error: errorMessage || `Worker exited ${code ?? signal}` };
|
||||
if (!stopped) logger.error('Training job failed', { error: state.error, stderr });
|
||||
} else {
|
||||
for (const [id, reason] of Object.entries(result.rejected || {})) ledger[id] = { rejected: reason };
|
||||
if (result.published) await adoptCheckpoint(root, result.published, result.promote);
|
||||
if (result.trained) {
|
||||
for (const id of result.accepted) ledger[id] = { rounds: (ledger[id]?.rounds || 0) + 1 };
|
||||
}
|
||||
state = { ...state, status: stopped ? 'stopped' : 'waiting', steps: result.steps ?? state.steps,
|
||||
completedAt: Date.now(), elapsedSeconds: result.elapsedSeconds ?? null, error: null,
|
||||
examples: result.examples ?? state.examples, loss: result.loss ?? null,
|
||||
diagnostics: result.diagnostics ?? state.diagnostics, publication: result.publication ?? state.publication, evaluation: result.evaluation ?? state.evaluation,
|
||||
reason: result.reason ?? 'Waiting for the next scheduled training job' };
|
||||
await saveLedger();
|
||||
logger.info('Training job completed', { ...state, model: result.published?.name || null });
|
||||
}
|
||||
} catch (error) {
|
||||
state = { ...state, status: 'error', error: error.message };
|
||||
logger.error('Training result persistence failed', { error: error.message });
|
||||
} finally {
|
||||
history.push({ at: Date.now(), status: state.status, steps: state.steps, loss: state.loss,
|
||||
losses: state.losses || null, evaluation: state.evaluation || null,
|
||||
elapsedSeconds: state.elapsedSeconds || null });
|
||||
if (history.length > 20) history.shift();
|
||||
leases.clear();
|
||||
child = null;
|
||||
nextRun = Date.now() + config.intervalSeconds * 1000;
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function cancel() {
|
||||
if (!child) return;
|
||||
const worker = child;
|
||||
worker.kill('SIGTERM');
|
||||
const timer = setTimeout(() => worker.kill('SIGKILL'), 5000);
|
||||
try { await completion; } finally { clearTimeout(timer); }
|
||||
}
|
||||
|
||||
return {
|
||||
leases, start, tick, cancel,
|
||||
getState: () => ({ ...state, history, nextAttemptAt: nextRun || null, leasedSessions: leases.size,
|
||||
visitedSessions: Object.values(ledger).filter((entry) => entry.rounds).length,
|
||||
skippedSessions: Object.values(ledger).filter((entry) => entry.rejected).length,
|
||||
rejectionReasons: [...new Set(Object.values(ledger).filter((entry) => entry.rejected).map((entry) => entry.rejected))].slice(0, 10) }),
|
||||
stop: async () => { stopped = true; await cancel(); },
|
||||
kill: () => child?.kill('SIGKILL'),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { createTrainer };
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Read closed recordings with causal state/action alignment and bounded memory."""
|
||||
import bisect
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
|
||||
from policy import SLOTS, VALUES, HEIGHT, WIDTH, encode_command, sensor_vector
|
||||
|
||||
|
||||
def load_events(directory):
|
||||
metadata = json.loads((directory / 'session.json').read_text())
|
||||
if (metadata.get('version') != 1 or not metadata.get('endedAt') or metadata.get('error')
|
||||
or metadata.get('droppedEvents') or metadata.get('reason') == 'interrupted'):
|
||||
raise ValueError('Incomplete/interrupted recording or lost events')
|
||||
if metadata.get('video', {}).get('origin') != 'unix':
|
||||
raise ValueError('Unsupported video time origin')
|
||||
video_result = metadata.get('video', {}).get('result') or {}
|
||||
if video_result.get('signal') == 'SIGKILL' or video_result.get('code') not in (0, 255, None):
|
||||
raise ValueError('Video recorder failed or was forcibly killed')
|
||||
event_file = directory / 'events.ndjson'
|
||||
if event_file.stat().st_size > 64 * 1024 * 1024:
|
||||
raise ValueError('Event file exceeds 64 MiB read bound')
|
||||
events = []
|
||||
with event_file.open() as stream:
|
||||
for line in stream:
|
||||
event = json.loads(line)
|
||||
if event['kind'] == 'captureGap':
|
||||
raise ValueError('Recording contains a capture gap')
|
||||
if not isinstance(event.get('ts'), (int, float)) or not math.isfinite(event['ts']):
|
||||
raise ValueError('Invalid event timestamp')
|
||||
events.append(event)
|
||||
if not events or not any(e['kind'] == 'sessionEnd' for e in events):
|
||||
raise ValueError('Missing completed event stream')
|
||||
# File order can differ across rotation buffering. Monotonic time detects
|
||||
# wall-clock steps; sorting Unix timestamps alone would hide those steps.
|
||||
timed = sorted(events, key=lambda e: int(e['monotonicNs']))
|
||||
origin = timed[0]
|
||||
for event in timed:
|
||||
drift = (event['ts'] - origin['ts']) - (int(event['monotonicNs']) - int(origin['monotonicNs'])) / 1e6
|
||||
if abs(drift) > 250:
|
||||
raise ValueError('Server wall clock shifted during recording')
|
||||
return metadata, sorted(events, key=lambda e: e['ts'])
|
||||
|
||||
|
||||
def prepare(directory, schema, stats):
|
||||
metadata, events = load_events(directory)
|
||||
requests = {}
|
||||
accepted = set()
|
||||
failed = set()
|
||||
sensors, dispatches, boundaries = [], [], []
|
||||
for event in events:
|
||||
data = event['data']
|
||||
if event['kind'] == 'sensor':
|
||||
sensors.append((event['ts'], sensor_vector(data.get('sensors') or {})))
|
||||
if event['kind'] in ('driver', 'turn:activeDriver', 'lock', 'private'):
|
||||
boundaries.append(event['ts'])
|
||||
if event['kind'] != 'command':
|
||||
continue
|
||||
if data.get('phase') == 'requested' and data.get('source') == 'client':
|
||||
requests[data.get('requestId')] = data
|
||||
if data.get('phase') == 'result' and data.get('result', {}).get('id'):
|
||||
accepted.add(data['result']['id'])
|
||||
if data.get('phase') == 'acknowledged' and data.get('acknowledgement', {}).get('error'):
|
||||
failed.add(data.get('commandId'))
|
||||
if data.get('phase') == 'dispatched':
|
||||
dispatches.append(data)
|
||||
if not sensors:
|
||||
raise ValueError('No sensor observations')
|
||||
candidate = copy.deepcopy(schema)
|
||||
history, targets, wheels = [], [], [(stamp, None) for stamp in boundaries]
|
||||
stats.update({'sensorFrames': len(sensors), 'dispatches': len(dispatches),
|
||||
'failedCommands': len(failed), 'sensorPresentFraction': np.mean([row[1][32:] for row in sensors], axis=0).tolist()})
|
||||
for data in dispatches:
|
||||
command = data.get('command') or {}
|
||||
request = requests.get(data.get('requestId'))
|
||||
human = (data.get('source') == 'client' and request and data.get('commandId') in accepted
|
||||
and data.get('commandId') not in failed)
|
||||
stamp = data['ts']
|
||||
if command.get('type') == 'drive':
|
||||
direct = command.get('driveDirect') or {}
|
||||
pair = [direct.get('left'), direct.get('right')]
|
||||
if human and all(isinstance(v, (int, float)) and math.isfinite(v) for v in pair):
|
||||
wheels.append((stamp, np.clip(np.array(pair, dtype=np.float32), -metadata.get('wheelSpeedLimit', 500), metadata.get('wheelSpeedLimit', 500)) / 500))
|
||||
else:
|
||||
wheels.append((stamp, None))
|
||||
elif command.get('type') in ('raw', 'reset', 'reboot', 'update', 'sensorStream'):
|
||||
# Raw OI can change mode or motion; do not assume the last wheel
|
||||
# command survives it. A later explicit human drive resolves intent.
|
||||
wheels.append((stamp, None))
|
||||
boundaries.append(stamp)
|
||||
encoded = encode_command(command, candidate, extend=True)
|
||||
if encoded is None:
|
||||
continue
|
||||
stamp = data['ts']
|
||||
history.append((stamp, *encoded))
|
||||
request = requests.get(data.get('requestId'))
|
||||
if (data.get('source') == 'client' and request
|
||||
and data.get('commandId') in accepted and data.get('commandId') not in failed):
|
||||
target_time = request['ts']
|
||||
if not 0 <= stamp - target_time <= 1000:
|
||||
raise ValueError('Invalid request/dispatch timing')
|
||||
targets.append((target_time, *encoded))
|
||||
else:
|
||||
# Server safety overrides belong in history, never demonstration
|
||||
# targets. Exclude windows around them rather than teaching inaction.
|
||||
boundaries.append(stamp)
|
||||
if not targets:
|
||||
raise ValueError('No accepted client operating commands')
|
||||
schema[:] = candidate
|
||||
return metadata, sensors, sorted(wheels, key=lambda x: x[0]), sorted(history, key=lambda x: x[0]), sorted(targets, key=lambda x: x[0]), sorted(boundaries)
|
||||
|
||||
|
||||
def windows(directory, schema, cancelled=lambda: False, stats=None):
|
||||
directory = Path(directory)
|
||||
stats = stats if stats is not None else {}
|
||||
metadata, sensors, wheels, history, targets, boundaries = prepare(directory, schema, stats)
|
||||
wheel_times = [row[0] for row in wheels]
|
||||
stats.update({'usableWindows': 0, 'movingWindows': 0, 'stoppedWindows': 0,
|
||||
'unknownWheelWindows': 0, 'staleSensorWindows': 0, 'boundaryWindows': 0,
|
||||
'ambiguousWindows': 0, 'videoFrames': 0, 'maxVideoGapMs': 0})
|
||||
sensor_times = [row[0] for row in sensors]
|
||||
target_times = [row[0] for row in targets]
|
||||
available = np.zeros(SLOTS, dtype=np.float32)
|
||||
for _, slot, _ in history:
|
||||
available[slot] = float(schema[slot]['template']['type'] != 'drive')
|
||||
command_values = np.zeros((SLOTS, VALUES), dtype=np.float32)
|
||||
command_times = np.full(SLOTS, -np.inf)
|
||||
cursor = 0
|
||||
recent = collections.deque(maxlen=32)
|
||||
last_sample, last_pts = -math.inf, -math.inf
|
||||
def frames():
|
||||
files = metadata['video'].get('files', ['video.mkv'])
|
||||
for name in files:
|
||||
if Path(name).name != name:
|
||||
raise ValueError('Invalid video segment path')
|
||||
with av.open(str(directory / name)) as container:
|
||||
stream = container.streams.video[0]
|
||||
stream.codec_context.thread_count = 1
|
||||
yield from container.decode(stream)
|
||||
|
||||
for frame in frames():
|
||||
if cancelled():
|
||||
return
|
||||
if frame.pts is None:
|
||||
raise ValueError('Video frame has no timestamp')
|
||||
stamp = float(frame.pts * frame.time_base) * 1000
|
||||
if not math.isfinite(stamp) or stamp < last_pts:
|
||||
raise ValueError('Invalid/nonmonotonic video timestamps')
|
||||
stats['videoFrames'] += 1
|
||||
if math.isfinite(last_pts):
|
||||
stats['maxVideoGapMs'] = max(stats['maxVideoGapMs'], stamp - last_pts)
|
||||
last_pts = stamp
|
||||
if not metadata['startedAt'] - 2000 <= stamp <= metadata['endedAt'] + 2000:
|
||||
raise ValueError('Video PTS is not aligned to session Unix time')
|
||||
if stamp < metadata.get('trainingStartedAt', metadata['startedAt']):
|
||||
continue
|
||||
if stamp - last_sample < 99:
|
||||
continue
|
||||
last_sample = stamp
|
||||
while cursor < len(history) and history[cursor][0] < stamp:
|
||||
when, slot, values = history[cursor]
|
||||
command_values[slot] = values
|
||||
command_times[slot] = when
|
||||
cursor += 1
|
||||
sensor_index = bisect.bisect_right(sensor_times, stamp) - 1
|
||||
if sensor_index < 0 or stamp - sensor_times[sensor_index] > 1000:
|
||||
stats['staleSensorWindows'] += 1
|
||||
recent.clear()
|
||||
continue
|
||||
seen = np.isfinite(command_times).astype(np.float32)
|
||||
ages = np.minimum((stamp - command_times) / 10000, 1).astype(np.float32)
|
||||
state = np.concatenate((sensors[sensor_index][1], command_values.flatten(), ages, seen))
|
||||
pixels = frame.to_ndarray(width=WIDTH, height=HEIGHT, format='rgb24').transpose(2, 0, 1).copy()
|
||||
recent.append((stamp, pixels, state))
|
||||
chosen = []
|
||||
for offset in (900, 600, 300, 0):
|
||||
options = [row for row in recent if row[0] <= stamp - offset + 0.01]
|
||||
if not options or stamp - offset - options[-1][0] > 200:
|
||||
break
|
||||
chosen.append(options[-1])
|
||||
if len(chosen) != 4 or stamp + 100 > metadata['endedAt']:
|
||||
continue
|
||||
if bisect.bisect_right(boundaries, stamp + 100) > bisect.bisect_left(boundaries, chosen[0][0]):
|
||||
stats['boundaryWindows'] += 1
|
||||
continue
|
||||
before = bisect.bisect_left(wheel_times, stamp) - 1
|
||||
after = bisect.bisect_right(wheel_times, stamp + 100) - 1
|
||||
if before < 0 or after < 0 or wheels[before][1] is None or wheels[after][1] is None:
|
||||
stats['unknownWheelWindows'] += 1
|
||||
continue
|
||||
wheel_target, previous_wheels = wheels[after][1], wheels[before][1]
|
||||
start = bisect.bisect_left(target_times, stamp)
|
||||
end = bisect.bisect_left(target_times, stamp + 100)
|
||||
actions = np.zeros(SLOTS, dtype=np.float32)
|
||||
values = np.zeros((SLOTS, VALUES), dtype=np.float32)
|
||||
ambiguous = False
|
||||
families = set()
|
||||
for _, slot, vector in targets[start:end]:
|
||||
template = schema[slot]['template']
|
||||
family = template['type']
|
||||
if family == 'drive':
|
||||
continue
|
||||
if family == 'peripheral':
|
||||
peripheral = template.get('peripheral') or {}
|
||||
family += ':' + str(peripheral.get('id')) + ':' + str(peripheral.get('control'))
|
||||
if ((actions[slot] and (not schema[slot]['fields'] or not np.array_equal(values[slot], vector)))
|
||||
or (not actions[slot] and family in families)):
|
||||
ambiguous = True
|
||||
break
|
||||
families.add(family)
|
||||
actions[slot], values[slot] = 1, vector
|
||||
if ambiguous:
|
||||
stats['ambiguousWindows'] += 1
|
||||
continue
|
||||
stats['usableWindows'] += 1
|
||||
stats['movingWindows' if np.max(np.abs(wheel_target)) > .02 else 'stoppedWindows'] += 1
|
||||
yield np.stack([row[1] for row in chosen]), np.stack([row[2] for row in chosen]), actions, values, available, wheel_target, previous_wheels
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live policy worker. JSON state on stdin, proposals on stdout; no rover writes."""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from policy import Policy, VERSION, SLOTS, VALUES, HEIGHT, WIDTH, encode_command, sensor_vector
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--model', required=True)
|
||||
parser.add_argument('--url', required=True)
|
||||
parser.add_argument('--threads', type=int, default=4)
|
||||
parser.add_argument('--threshold', type=float, default=0.7)
|
||||
args = parser.parse_args()
|
||||
torch.set_num_threads(args.threads)
|
||||
torch.set_num_interop_threads(1)
|
||||
directory = Path(args.model)
|
||||
metadata = json.loads((directory / 'model.json').read_text())
|
||||
if metadata['specification']['version'] != VERSION:
|
||||
raise ValueError('Model specification is incompatible')
|
||||
if hashlib.sha256((directory / 'weights.pt').read_bytes()).hexdigest() != metadata['sha256']:
|
||||
raise ValueError('Model weights checksum mismatch')
|
||||
schema = metadata['specification']['commands']
|
||||
model = Policy().eval()
|
||||
model.load_state_dict(torch.load(directory / 'weights.pt', map_location='cpu', weights_only=True))
|
||||
observations = collections.deque(maxlen=2048)
|
||||
mutex = threading.Lock()
|
||||
closed = threading.Event()
|
||||
|
||||
def read_input():
|
||||
try:
|
||||
for line in sys.stdin:
|
||||
event = json.loads(line)
|
||||
with mutex:
|
||||
observations.append(event)
|
||||
finally:
|
||||
closed.set()
|
||||
|
||||
threading.Thread(target=read_input, daemon=True).start()
|
||||
values = np.zeros((SLOTS, VALUES), dtype=np.float32)
|
||||
last_commands = np.full(SLOTS, -np.inf)
|
||||
sensors, sensor_time = {}, 0
|
||||
history = collections.deque(maxlen=32)
|
||||
latched = set()
|
||||
sampled = 0
|
||||
with av.open(args.url, options={'rtsp_transport': 'tcp', 'timeout': '3000000',
|
||||
'use_wallclock_as_timestamps': '1'}) as video:
|
||||
stream = video.streams.video[0]
|
||||
stream.codec_context.thread_count = 1
|
||||
for frame in video.decode(stream):
|
||||
if closed.is_set():
|
||||
return
|
||||
if frame.pts is None:
|
||||
continue
|
||||
stamp = float(frame.pts * frame.time_base) * 1000
|
||||
now = time.time() * 1000
|
||||
if not math.isfinite(stamp) or abs(now - stamp) > 2000:
|
||||
raise ValueError('Video time is stale or not aligned to server time')
|
||||
if stamp - sampled < 99:
|
||||
continue
|
||||
sampled = stamp
|
||||
with mutex:
|
||||
while observations and observations[0]['ts'] < stamp:
|
||||
event = observations.popleft()
|
||||
if event['kind'] == 'settings':
|
||||
args.threshold = max(.05, min(.99, float(event['threshold'])))
|
||||
elif event['kind'] == 'sensor':
|
||||
sensors, sensor_time = event['data'], event['ts']
|
||||
elif event['kind'] == 'command':
|
||||
try:
|
||||
encoded = encode_command(event['data'], schema)
|
||||
if encoded:
|
||||
slot, vector = encoded
|
||||
values[slot], last_commands[slot] = vector, event['ts']
|
||||
except ValueError:
|
||||
pass # Unseen manual controls do not stop autonomy.
|
||||
if stamp - sensor_time > 1000:
|
||||
history.clear()
|
||||
continue
|
||||
state = np.concatenate((sensor_vector(sensors), values.flatten(),
|
||||
np.minimum((stamp - last_commands) / 10000, 1).astype(np.float32),
|
||||
np.isfinite(last_commands).astype(np.float32)))
|
||||
pixels = frame.to_ndarray(width=WIDTH, height=HEIGHT, format='rgb24').transpose(2, 0, 1).copy()
|
||||
history.append((stamp, pixels, state))
|
||||
selected = []
|
||||
for offset in (900, 600, 300, 0):
|
||||
candidates = [entry for entry in history if entry[0] <= stamp - offset + .01]
|
||||
if not candidates or stamp - offset - candidates[-1][0] > 200:
|
||||
break
|
||||
selected.append(candidates[-1])
|
||||
if len(selected) != 4:
|
||||
continue
|
||||
before = time.perf_counter()
|
||||
with torch.inference_mode():
|
||||
logits, numeric, wheel_output = model(torch.from_numpy(np.stack([row[1] for row in selected])).unsqueeze(0).float() / 255,
|
||||
torch.from_numpy(np.stack([row[2] for row in selected])).unsqueeze(0))
|
||||
probabilities = logits[0].sigmoid().tolist()
|
||||
predictions = numeric[0].tolist()
|
||||
wheel_speeds = [round(value * 500) for value in wheel_output[0].tolist()]
|
||||
if not all(math.isfinite(value) for value in probabilities):
|
||||
raise ValueError('Nonfinite model output')
|
||||
families = {}
|
||||
active = set()
|
||||
proposals = []
|
||||
for slot, descriptor in enumerate(schema):
|
||||
if descriptor['template']['type'] == 'drive':
|
||||
continue
|
||||
command = json.loads(json.dumps(descriptor['template']))
|
||||
family = command['type']
|
||||
if family == 'peripheral':
|
||||
family += ':' + str(command['peripheral']['id']) + ':' + str(command['peripheral']['control'])
|
||||
for index, field in enumerate(descriptor['fields']):
|
||||
number = predictions[slot][index]
|
||||
if not math.isfinite(number):
|
||||
raise ValueError('Nonfinite control value')
|
||||
number = math.tan(max(-.98, min(.98, number)) * math.pi / 2) * field['scale']
|
||||
target = command
|
||||
for key in field['path'][:-1]:
|
||||
target = target[key]
|
||||
target[field['path'][-1]] = round(number) if field['integer'] else number
|
||||
proposals.append({'slot': slot, 'score': probabilities[slot], 'command': command,
|
||||
'reason': 'Below threshold'})
|
||||
if probabilities[slot] < args.threshold:
|
||||
continue
|
||||
active.add(slot)
|
||||
discrete = command['type'] in ('headlight', 'laser', 'horn', 'song', 'raw')
|
||||
if family not in families or probabilities[slot] > families[family][0]:
|
||||
families[family] = (probabilities[slot], command, slot, discrete)
|
||||
latched.intersection_update(active)
|
||||
# Choose the winning action before latching, so a held toggle cannot
|
||||
# expose a competing lower-confidence toggle on the following tick.
|
||||
commands = [command for _, command, slot, discrete in families.values()
|
||||
if not discrete or slot not in latched]
|
||||
winners = {row[2]: row for row in families.values()}
|
||||
for proposal in proposals:
|
||||
slot = proposal['slot']
|
||||
if slot in active:
|
||||
proposal['reason'] = ('Competing action' if slot not in winners else
|
||||
'Held discrete action' if winners[slot][3] and slot in latched else 'Proposed')
|
||||
for _, _, slot, discrete in families.values():
|
||||
if discrete:
|
||||
latched.add(slot)
|
||||
commands.insert(0, {'type': 'drive', 'driveDirect': dict(zip(('left', 'right'), wheel_speeds))})
|
||||
print(json.dumps({'kind': 'prediction', 'frameAt': stamp, 'sensorAt': sensor_time,
|
||||
'latencyMs': (time.perf_counter() - before) * 1000,
|
||||
'wheelSpeeds': wheel_speeds, 'sensorPresent': sensor_vector(sensors)[32:].tolist(),
|
||||
'commands': commands, 'proposals': proposals, 'threshold': args.threshold}, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print(json.dumps({'kind': 'error', 'message': str(error)}), flush=True)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Versioned, fixed-size vision/control policy and command encoding.
|
||||
|
||||
Slots describe command shapes, not arbitrary executable instructions. Numeric
|
||||
leaves are regressed; string/bool/list values remain exact template constants.
|
||||
No command is sent by this module. Execution must use server authorization.
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
VERSION = 2
|
||||
SLOTS = 64
|
||||
VALUES = 8
|
||||
FRAMES = 4
|
||||
HEIGHT, WIDTH = 120, 160
|
||||
OPERATING_TYPES = {"drive", "motors", "servo", "headlight", "laser", "horn", "peripheral", "song", "raw"}
|
||||
SENSORS = [
|
||||
("bumpsAndWheelDrops.bumpLeft", 1), ("bumpsAndWheelDrops.bumpRight", 1),
|
||||
("bumpsAndWheelDrops.wheelDropLeft", 1), ("bumpsAndWheelDrops.wheelDropRight", 1),
|
||||
("cliffLeft", 1), ("cliffFrontLeft", 1), ("cliffFrontRight", 1), ("cliffRight", 1),
|
||||
("wall", 1), ("virtualWall", 1), ("wallSignal", 4096),
|
||||
("voltageMv", 16000), ("currentMa", 3000), ("batteryChargeMah", 4000),
|
||||
("batteryCapacityMah", 4000), ("batteryTemperatureC", 50),
|
||||
("chargingState.code", 5), ("chargingSources.homeBase", 1), ("oiMode.code", 4),
|
||||
("wheelSpeedsMmPerSecond.left", 500), ("wheelSpeedsMmPerSecond.right", 500),
|
||||
("wheelLeftCurrentMa", 2000), ("wheelRightCurrentMa", 2000),
|
||||
("mainBrushCurrentMa", 2000), ("sideBrushCurrentMa", 2000),
|
||||
("lightBumpLeftSignal", 4096), ("lightBumpRightSignal", 4096),
|
||||
("cliffLeftSignal", 4096), ("cliffRightSignal", 4096),
|
||||
("distanceMm", 500), ("angleDeg", 180), ("stasis", 1),
|
||||
]
|
||||
STATE_SIZE = len(SENSORS) * 2 + SLOTS * (VALUES + 2)
|
||||
|
||||
|
||||
def normalize(value, scale):
|
||||
return math.atan(float(value) / scale) * 2 / math.pi
|
||||
|
||||
|
||||
def sensor_vector(sensors):
|
||||
values, present = [], []
|
||||
for name, scale in SENSORS:
|
||||
value = sensors
|
||||
for key in name.split('.'):
|
||||
value = value.get(key) if isinstance(value, dict) else None
|
||||
known = isinstance(value, (int, float)) and math.isfinite(value)
|
||||
values.append(normalize(value, scale) if known else 0)
|
||||
present.append(float(known))
|
||||
return np.array(values + present, dtype=np.float32)
|
||||
|
||||
|
||||
def describe_command(command):
|
||||
if command.get('type') not in OPERATING_TYPES:
|
||||
return None
|
||||
if len(json.dumps(command)) > 4096:
|
||||
raise ValueError('Operating command exceeds 4096-byte schema limit')
|
||||
values, fields = [], []
|
||||
|
||||
def visit(value, path=()):
|
||||
if isinstance(value, dict):
|
||||
return {key: visit(item, (*path, key)) for key, item in sorted(value.items())}
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
if not math.isfinite(value) or len(values) >= VALUES:
|
||||
raise ValueError('Nonfinite command or too many numeric control fields')
|
||||
scale = {'drive': 500, 'motors': 255, 'servo': 180, 'peripheral': 255,
|
||||
'horn': 1000, 'song': 127}.get(command['type'], 1)
|
||||
fields.append({'path': list(path), 'scale': scale, 'integer': isinstance(value, int)})
|
||||
values.append(normalize(value, scale))
|
||||
return {'$number': len(values) - 1}
|
||||
# Arrays (e.g. songs) are discrete exact demonstrations, not variable
|
||||
# length regression outputs. Never reinterpret raw OI bytes as numbers.
|
||||
return copy.deepcopy(value)
|
||||
|
||||
template = visit(command)
|
||||
return {'template': template, 'fields': fields}, values
|
||||
|
||||
|
||||
def encode_command(command, schema, extend=False):
|
||||
described = describe_command(command)
|
||||
if described is None:
|
||||
return None
|
||||
descriptor, values = described
|
||||
try:
|
||||
slot = schema.index(descriptor)
|
||||
except ValueError:
|
||||
if not extend:
|
||||
raise ValueError('Command shape missing from model schema')
|
||||
if len(schema) >= SLOTS:
|
||||
raise ValueError('64 command shapes exhausted; recording skipped, not silently truncated')
|
||||
slot = len(schema)
|
||||
schema.append(descriptor)
|
||||
vector = np.zeros(VALUES, dtype=np.float32)
|
||||
vector[:len(values)] = values
|
||||
return slot, vector
|
||||
|
||||
|
||||
class Policy(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.vision = nn.Sequential(
|
||||
nn.Conv2d(3, 24, 5, 2, 2), nn.ReLU(),
|
||||
nn.Conv2d(24, 48, 3, 2, 1), nn.ReLU(),
|
||||
nn.Conv2d(48, 96, 3, 2, 1), nn.ReLU(),
|
||||
nn.Conv2d(96, 128, 3, 2, 1), nn.ReLU(),
|
||||
nn.AdaptiveAvgPool2d((3, 4)), nn.Flatten(),
|
||||
nn.Linear(128 * 3 * 4, 128), nn.ReLU(),
|
||||
)
|
||||
self.history = nn.GRU(128 + STATE_SIZE, 128, batch_first=True)
|
||||
self.wheels = nn.Linear(128, 2)
|
||||
self.actions = nn.Linear(128, SLOTS)
|
||||
self.values = nn.Linear(128, SLOTS * VALUES)
|
||||
|
||||
def forward(self, images, state):
|
||||
batch, frames = images.shape[:2]
|
||||
encoded = self.vision(images.reshape(-1, 3, HEIGHT, WIDTH)).reshape(batch, frames, -1)
|
||||
history, _ = self.history(torch.cat((encoded, state), dim=-1))
|
||||
last = history[:, -1]
|
||||
return self.actions(last), self.values(last).reshape(batch, SLOTS, VALUES).tanh(), self.wheels(last).tanh()
|
||||
|
||||
|
||||
def specification(schema):
|
||||
return {'version': VERSION, 'architecture': 'spatial-cnn-gru-wheels-v2',
|
||||
'frames': FRAMES, 'height': HEIGHT, 'width': WIDTH,
|
||||
'historyStrideMs': 300, 'decisionIntervalMs': 100,
|
||||
'stateSize': STATE_SIZE, 'sensorFields': SENSORS,
|
||||
'normalization': '2/pi * atan(value/scale); missing sensors have mask=0',
|
||||
'maxCommandShapes': SLOTS, 'numericFieldsPerShape': VALUES,
|
||||
'commands': schema, 'actionSemantics': 'Continuous wheels at +100ms; accessories are events in the next 100ms',
|
||||
'wheelScaleMmPerSecond': 500, 'wheelOrder': ['left', 'right'],
|
||||
'wheelCommandExpiryMs': None, 'unknownWheelIntent': 'Excluded until next accepted human drive',
|
||||
'timing': 'Server-receipt alignment; browser display latency unmeasured'}
|
||||
@@ -0,0 +1,4 @@
|
||||
--extra-index-url https://download.pytorch.org/whl/cpu
|
||||
torch==2.14.0+cpu
|
||||
numpy==2.5.3
|
||||
av==18.1.0
|
||||
@@ -0,0 +1,352 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One finite CPU training job; Node owns scheduling, leases, and shutdown.
|
||||
|
||||
Usage: python train.py --job /path/to/job.json
|
||||
Dependencies are packaged at image build time, never installed at runtime.
|
||||
"""
|
||||
import argparse
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import resource
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
def emit(kind, **data):
|
||||
print(json.dumps({'kind': kind, **data}, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
def atomic_torch_save(torch, value, path):
|
||||
temporary = path.with_suffix('.tmp')
|
||||
with temporary.open('wb') as stream:
|
||||
torch.save(value, stream)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--job')
|
||||
parser.add_argument('--check-dependencies', action='store_true', help='Describe the installed production policy and exit without creating storage')
|
||||
args = parser.parse_args()
|
||||
# --help works even before Python dependencies are installed.
|
||||
import numpy as np
|
||||
import av
|
||||
import torch
|
||||
from torch.nn import functional as functional
|
||||
from dataset import windows
|
||||
from policy import Policy, VERSION, SLOTS, VALUES, specification
|
||||
|
||||
if args.check_dependencies:
|
||||
emit('dependencies', torch=str(torch.__version__), numpy=str(np.__version__), av=str(av.__version__),
|
||||
parameters=sum(p.numel() for p in Policy().parameters()), specification=specification([]))
|
||||
return
|
||||
if not args.job:
|
||||
parser.error('--job is required unless --check-dependencies is used')
|
||||
|
||||
job = json.loads(Path(args.job).read_text())
|
||||
root = Path(job['root'])
|
||||
config = job['config']
|
||||
torch.set_num_threads(config['threads'])
|
||||
torch.set_num_interop_threads(1)
|
||||
# Training yields CPU priority to the Node/media processes. Thread count is
|
||||
# an additional bound, not a claim to reserve specific physical cores.
|
||||
os.nice(10)
|
||||
stopped = False
|
||||
|
||||
def stop(_signal, _frame):
|
||||
nonlocal stopped
|
||||
stopped = True
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
started = time.monotonic()
|
||||
cancelled = lambda: stopped or time.monotonic() - started >= config['maxJobSeconds']
|
||||
model = Policy().cpu()
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=0.0001)
|
||||
# A fresh training lineage prevents previously trained footage leaking into validation.
|
||||
resume_path = root / 'training' / 'resume-v3.pt'
|
||||
schema, steps, examples, last_published = [], 0, 0, 0
|
||||
trained_rovers = set()
|
||||
control_profiles = []
|
||||
distinct_windows = {}
|
||||
published_windows = 0
|
||||
if resume_path.exists():
|
||||
saved = torch.load(resume_path, map_location='cpu', weights_only=True)
|
||||
if saved['version'] != VERSION:
|
||||
raise ValueError('Resume schema incompatible; refusing to reinterpret existing weights')
|
||||
model.load_state_dict(saved['weights'])
|
||||
optimizer.load_state_dict(saved['optimizer'])
|
||||
schema = saved['schema']
|
||||
steps, examples = saved['steps'], saved['examples']
|
||||
last_published = saved['lastPublished']
|
||||
trained_rovers = set(saved['trainedRovers'])
|
||||
control_profiles = saved.get('controlProfiles', [])
|
||||
distinct_windows = saved.get('distinctWindows', {})
|
||||
published_windows = saved.get('publishedWindows', 0)
|
||||
cpu_started = time.process_time()
|
||||
decode_started = time.monotonic()
|
||||
dataset_stats = []
|
||||
initial_steps = steps
|
||||
pool, validation, accepted, rejected = [], [], [], {}
|
||||
observed_windows = {}
|
||||
sessions = list(job['sessions'])
|
||||
random.shuffle(sessions)
|
||||
is_held_out = lambda session_id: int(hashlib.sha256(session_id.encode()).hexdigest()[:8], 16) % 5 == 0
|
||||
sessions.sort(key=is_held_out)
|
||||
per_session = max(1, config['maxSamples'] // max(1, len(sessions)))
|
||||
for session_id in sessions:
|
||||
if cancelled():
|
||||
break
|
||||
candidate_schema = copy.deepcopy(schema)
|
||||
samples, seen = [], 0
|
||||
stats = {}
|
||||
buckets = {key: [] for key in ('stopped', 'forward', 'reverse', 'turn')}
|
||||
counts = dict.fromkeys(buckets, 0)
|
||||
held_out = is_held_out(session_id)
|
||||
try:
|
||||
directory = root / 'recordings' / session_id
|
||||
for sample in windows(directory, candidate_schema, cancelled, stats):
|
||||
seen += 1
|
||||
# Keep evaluation naturally distributed. Training reserves space
|
||||
# for ordinary turns/reversing so idle or straight motion cannot
|
||||
# consume the entire sample budget. No scripted driving required.
|
||||
if held_out:
|
||||
reservoir, count, capacity = samples, seen, per_session
|
||||
else:
|
||||
left, right = sample[5]
|
||||
category = ('stopped' if max(abs(left), abs(right)) <= .02 else
|
||||
'turn' if abs(left - right) > .1 else
|
||||
'reverse' if left + right < 0 else 'forward')
|
||||
counts[category] += 1
|
||||
reservoir, count, capacity = buckets[category], counts[category], max(1, per_session // 4)
|
||||
if len(reservoir) < capacity:
|
||||
reservoir.append(sample)
|
||||
else:
|
||||
index = random.randrange(count)
|
||||
if index < capacity:
|
||||
reservoir[index] = sample
|
||||
if not held_out:
|
||||
samples = [sample for bucket in buckets.values() for sample in bucket]
|
||||
if cancelled():
|
||||
break # Do not call a partially decoded recording validated.
|
||||
if not samples:
|
||||
raise ValueError('No complete, aligned observation/action windows')
|
||||
# Entire sessions stay in one partition across all jobs. Neighboring
|
||||
# video windows must never straddle training and validation.
|
||||
held_out = is_held_out(session_id)
|
||||
if held_out:
|
||||
if candidate_schema != schema:
|
||||
dataset_stats.append({**stats, 'partition': 'heldOut', 'sampledWindows': len(samples),
|
||||
'skipReason': 'Vocabulary absent from training data'})
|
||||
continue # Retry after training data introduces these command shapes.
|
||||
validation.extend(samples)
|
||||
else:
|
||||
schema = candidate_schema
|
||||
pool.extend(samples)
|
||||
observed_windows[session_id] = seen
|
||||
accepted.append(session_id)
|
||||
metadata = json.loads((directory / 'session.json').read_text())
|
||||
if not held_out:
|
||||
trained_rovers.add(str(metadata['roverId']))
|
||||
profile = metadata.get('controlProfile')
|
||||
if not held_out and profile is not None and profile not in control_profiles:
|
||||
control_profiles.append(profile)
|
||||
stats.update({'partition': 'heldOut' if held_out else 'training', 'sampledWindows': len(samples)})
|
||||
dataset_stats.append(stats)
|
||||
emit('dataset', sessionId=session_id, windows=seen, sampled=len(samples), diagnostics=stats)
|
||||
except (ValueError, KeyError, TypeError, OSError, EOFError, IndexError, OverflowError, av.error.FFmpegError) as error:
|
||||
rejected[session_id] = str(error)[:500]
|
||||
emit('skipped', sessionId=session_id, reason=rejected[session_id])
|
||||
if len(pool) < config['minimumSamples'] or not any(np.max(np.abs(sample[5])) > .02 for sample in pool) or cancelled():
|
||||
emit('complete', trained=False, accepted=[], rejected=rejected, samples=len(pool),
|
||||
diagnostics={'dataset': dataset_stats, 'decodeSeconds': time.monotonic() - decode_started},
|
||||
reason='Stopped/time limit' if cancelled() else 'Waiting for enough training-partition samples with human actions')
|
||||
return
|
||||
|
||||
decode_seconds = time.monotonic() - decode_started
|
||||
train_started = time.monotonic()
|
||||
numeric_mask = torch.zeros(SLOTS, VALUES)
|
||||
for slot, descriptor in enumerate(schema):
|
||||
numeric_mask[slot, :len(descriptor['fields'])] = 1
|
||||
# Balance sparse events without assuming that every time step should fire
|
||||
# every control. This is training loss, not a driving-quality evaluation.
|
||||
positives = np.sum([sample[2] for sample in pool], axis=0)
|
||||
opportunities = np.sum([sample[4] for sample in pool], axis=0)
|
||||
positive_weight = torch.from_numpy(np.clip((opportunities - positives) / np.maximum(positives, 1), 1, 20)).float()
|
||||
losses = []
|
||||
model.train()
|
||||
for _ in range(config['passesPerJob']):
|
||||
random.shuffle(pool)
|
||||
for offset in range(0, len(pool), config['batchSize']):
|
||||
if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
|
||||
break
|
||||
batch = pool[offset:offset + config['batchSize']]
|
||||
images, states, targets, values, available, wheel_targets, previous_wheels = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
|
||||
images = images.float().div_(255)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
action_logits, predictions, wheel_predictions = model(images, states)
|
||||
event_loss = functional.binary_cross_entropy_with_logits(action_logits, targets,
|
||||
pos_weight=positive_weight, reduction='none')
|
||||
event_loss = (event_loss * available).sum() / available.sum().clamp_min(1)
|
||||
mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
|
||||
value_loss = ((predictions - values).square() * mask).sum() / mask.sum().clamp_min(1)
|
||||
wheel_loss = functional.smooth_l1_loss(wheel_predictions, wheel_targets)
|
||||
loss = wheel_loss + event_loss + value_loss
|
||||
if not torch.isfinite(loss):
|
||||
raise ValueError('Nonfinite training loss; checkpoint not saved')
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
|
||||
optimizer.step()
|
||||
steps += 1
|
||||
examples += len(batch)
|
||||
losses.append(loss.item())
|
||||
if steps % 10 == 0:
|
||||
emit('progress', steps=steps, examples=examples, loss=loss.item(),
|
||||
losses={'wheels': wheel_loss.item(), 'events': event_loss.item(), 'accessoryValues': value_loss.item()})
|
||||
if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
|
||||
break
|
||||
if steps == initial_steps:
|
||||
emit('complete', trained=False, accepted=[], rejected=rejected, reason='No training steps completed')
|
||||
return
|
||||
|
||||
training_seconds = time.monotonic() - train_started
|
||||
evaluation_started = time.monotonic()
|
||||
distinct_windows.update(observed_windows)
|
||||
unique_count = sum(distinct_windows.values())
|
||||
evaluation = {'status': 'waiting for held-out sessions', 'windows': len(validation)}
|
||||
if validation and not cancelled():
|
||||
model.eval()
|
||||
true_positive = predicted_positive = actual_positive = 0
|
||||
value_error = value_count = 0
|
||||
evaluated = 0
|
||||
wheel_error = stopped_error = persistence_error = 0
|
||||
changed_error = changed_persistence = changed_count = moving_count = 0
|
||||
with torch.inference_mode():
|
||||
for offset in range(0, len(validation), config['batchSize']):
|
||||
if cancelled():
|
||||
break
|
||||
batch = validation[offset:offset + config['batchSize']]
|
||||
images, states, targets, values, available, wheel_targets, previous_wheels = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
|
||||
logits, predictions, wheel_predictions = model(images.float() / 255, states)
|
||||
predicted = (logits.sigmoid() >= job['actionThreshold']) & available.bool()
|
||||
actual = targets.bool() & available.bool()
|
||||
true_positive += (predicted & actual).sum().item()
|
||||
predicted_positive += predicted.sum().item()
|
||||
actual_positive += actual.sum().item()
|
||||
mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
|
||||
value_error += ((predictions - values).square() * mask).sum().item()
|
||||
value_count += mask.sum().item()
|
||||
errors = (wheel_predictions - wheel_targets).abs().mean(dim=1) * 500
|
||||
persistence = (previous_wheels - wheel_targets).abs().mean(dim=1) * 500
|
||||
changed = (previous_wheels - wheel_targets).abs().amax(dim=1) > .02
|
||||
changed_error += errors[changed].sum().item()
|
||||
changed_persistence += persistence[changed].sum().item()
|
||||
changed_count += changed.sum().item()
|
||||
moving_count += (wheel_targets.abs().amax(dim=1) > .02).sum().item()
|
||||
wheel_error += errors.sum().item()
|
||||
stopped_error += wheel_targets.abs().mean(dim=1).sum().item() * 500
|
||||
persistence_error += persistence.sum().item()
|
||||
evaluated += len(batch)
|
||||
precision = true_positive / max(1, predicted_positive)
|
||||
recall = true_positive / max(1, actual_positive)
|
||||
mse = value_error / max(1, value_count)
|
||||
wheel_mae = wheel_error / max(1, evaluated)
|
||||
stopped_mae = stopped_error / max(1, evaluated)
|
||||
persistence_mae = persistence_error / max(1, evaluated)
|
||||
changed_mae = changed_error / max(1, changed_count)
|
||||
changed_baseline = changed_persistence / max(1, changed_count)
|
||||
# Check control changes separately: copying the previous speed can look
|
||||
# excellent on long straight runs without learning to steer or stop.
|
||||
wheels_pass = (evaluated == len(validation) and evaluated >= 128 and moving_count >= 32
|
||||
and changed_count >= 32 and wheel_mae < stopped_mae
|
||||
and wheel_mae <= persistence_mae and changed_mae < changed_baseline)
|
||||
accessories_pass = (predicted_positive == 0 if actual_positive == 0 else
|
||||
actual_positive >= 32 and precision >= .5 and recall >= .5)
|
||||
passed = wheels_pass and accessories_pass and (value_count == 0 or mse <= .1)
|
||||
evaluation = {'status': 'passed imitation gate' if passed else 'imitation gate not met',
|
||||
'windows': evaluated, 'precision': precision, 'recall': recall,
|
||||
'numericMse': mse, 'positiveActions': actual_positive,
|
||||
'wheelMaeMmPerSecond': wheel_mae, 'stoppedBaselineMaeMmPerSecond': stopped_mae,
|
||||
'previousSpeedBaselineMaeMmPerSecond': persistence_mae,
|
||||
'changedWheelMaeMmPerSecond': changed_mae, 'changedBaselineMaeMmPerSecond': changed_baseline,
|
||||
'changedWindows': changed_count, 'movingWindows': moving_count,
|
||||
'wheelsPassed': wheels_pass, 'accessoriesPassed': accessories_pass}
|
||||
evaluation_seconds = time.monotonic() - evaluation_started
|
||||
publication = {'distinctMinutes': unique_count / 600,
|
||||
'newMinutes': (unique_count - published_windows) / 600,
|
||||
'minimumMinutes': config['minimumDrivingMinutes'],
|
||||
'requiredNewMinutes': config['newDrivingMinutesPerModel']}
|
||||
reasons = []
|
||||
if publication['distinctMinutes'] < config['minimumDrivingMinutes']:
|
||||
reasons.append('Collecting distinct driving footage')
|
||||
if publication['newMinutes'] < config['newDrivingMinutesPerModel']:
|
||||
reasons.append('Waiting for new driving footage since last publication')
|
||||
if steps - last_published < config['checkpointEverySteps']:
|
||||
reasons.append('Training steps below publication interval')
|
||||
if evaluation['status'] != 'passed imitation gate':
|
||||
reasons.append(evaluation['status'])
|
||||
if cancelled():
|
||||
reasons.append('Job time budget reached')
|
||||
publication['reason'] = '; '.join(reasons) or 'Latest updated'
|
||||
save_started = time.monotonic()
|
||||
published = None
|
||||
# Every completed job exports an immutable candidate; Node owns promotion and retention.
|
||||
adjectives = ['amber', 'curious', 'gentle', 'quiet', 'silver', 'bright', 'merry', 'sleepy']
|
||||
animals = ['otter', 'finch', 'badger', 'robin', 'fox', 'heron', 'marten', 'wren']
|
||||
model_id = str(uuid.uuid4())
|
||||
name = f'{random.choice(adjectives)}-{random.choice(animals)}-{model_id[:8]}'
|
||||
checkpoint_root = root / 'training' / 'checkpoints'
|
||||
checkpoint_root.mkdir(exist_ok=True)
|
||||
staging = checkpoint_root / f'.{model_id}.pending'
|
||||
staging.mkdir()
|
||||
weights_path = staging / 'weights.pt'
|
||||
atomic_torch_save(torch, model.state_dict(), weights_path)
|
||||
published = {
|
||||
'id': model_id, 'name': name, 'createdAt': int(time.time() * 1000),
|
||||
'trainingSteps': steps, 'examplesProcessed': examples,
|
||||
'parameters': sum(p.numel() for p in model.parameters()),
|
||||
'weightBytes': weights_path.stat().st_size,
|
||||
'sha256': hashlib.sha256(weights_path.read_bytes()).hexdigest(),
|
||||
'specification': specification(schema), 'trainedRovers': sorted(trained_rovers),
|
||||
'controlProfiles': control_profiles,
|
||||
'sourceSessions': accepted, 'trainingLoss': sum(losses) / len(losses),
|
||||
'evaluation': evaluation['status'], 'validation': evaluation, 'distinctMinutes': publication['distinctMinutes'], 'dtype': 'float32', 'torchVersion': str(torch.__version__),
|
||||
}
|
||||
(staging / 'model.json').write_text(json.dumps(published, indent=2, allow_nan=False))
|
||||
# A catalog reader sees both weights and metadata, or neither. Existing
|
||||
# versions are never overwritten or deleted by the trainer.
|
||||
staging.rename(checkpoint_root / model_id)
|
||||
if not reasons:
|
||||
last_published = steps
|
||||
published_windows = unique_count
|
||||
atomic_torch_save(torch, {
|
||||
'version': VERSION, 'weights': model.state_dict(), 'optimizer': optimizer.state_dict(),
|
||||
'schema': schema, 'steps': steps, 'examples': examples,
|
||||
'lastPublished': last_published, 'trainedRovers': sorted(trained_rovers),
|
||||
'controlProfiles': control_profiles, 'distinctWindows': distinct_windows,
|
||||
'publishedWindows': published_windows,
|
||||
}, resume_path)
|
||||
diagnostics = {'dataset': dataset_stats, 'decodeSeconds': decode_seconds,
|
||||
'trainingSeconds': training_seconds, 'evaluationSeconds': evaluation_seconds, 'checkpointSeconds': time.monotonic() - save_started,
|
||||
'cpuSeconds': time.process_time() - cpu_started,
|
||||
'peakRssMiB': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
|
||||
'dependencies': {'torch': str(torch.__version__), 'numpy': str(np.__version__), 'av': str(av.__version__)},
|
||||
'policy': specification(schema), 'browserVideoLatencyMs': None}
|
||||
emit('complete', trained=True, accepted=accepted, rejected=rejected,
|
||||
steps=steps, examples=examples, loss=sum(losses) / len(losses),
|
||||
published=published, promote=not reasons, publication=publication, evaluation=evaluation, diagnostics=diagnostics, elapsedSeconds=time.monotonic() - started)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
emit('error', message=str(error))
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user