diff --git a/plans/rover learning service.md b/plans/rover learning service.md index 4914fc00..0bfe8b3c 100644 --- a/plans/rover learning service.md +++ b/plans/rover learning service.md @@ -821,3 +821,17 @@ evaluation/cache changes remain proposals, not part of this repair. stream is available here; the race is fixed by inspection but cannot yet be confirmed as the cause of the earlier production events. Previously discarded error details cannot be reconstructed. No new tests or processes were created. + +### Deliberate held-out selection + +Latest IPI authorizes the scheduler correction only. Randomly shuffled eligible +sessions are now partitioned with exactly the worker's SHA256 first-eight-hex +modulo-five rule. Jobs select up to three training sessions and one held-out +session when both partitions exist; otherwise up to four training sessions. +No worker starts with held-out sessions alone. Existing split assignments, model +weights, and training progress are unchanged. Diagnostics expose timestamped +available/selected partition counts and the reason when evaluation cannot be +scheduled. Selection guarantees a held-out recording, not usable evaluation +windows: decoding/vocabulary/sample checks still apply in the worker. +Node syntax checks and diff whitespace checks passed; actual job selection on +production recordings remains unverified. No new tests or background processes. diff --git a/server/src/services/roverLearningService/diagnostics.js b/server/src/services/roverLearningService/diagnostics.js index 0134639e..15ce36f4 100644 --- a/server/src/services/roverLearningService/diagnostics.js +++ b/server/src/services/roverLearningService/diagnostics.js @@ -40,6 +40,7 @@ function report(state, controller) { sessions: state.recording.sessions.map((item) => ({ startedAtUnixMs: item.startedAt, status: item.video, droppedEvents: item.droppedEvents, error: redact(item.error), failureDetails: item.failureDetails || null, stderr: redact(item.stderr), errorCategory: errorCategory(item.error) })) }, training: { status: training.status, steps: training.steps, examples: training.examples, + selection: training.selection || null, loss: training.loss, losses: training.losses || null, publication: training.publication || null, evaluation: training.evaluation || null, dataset: metrics.dataset || training.dataset || null, resources: { decodeSeconds: metrics.decodeSeconds ?? null, trainingSeconds: metrics.trainingSeconds ?? null, diff --git a/server/src/services/roverLearningService/training.js b/server/src/services/roverLearningService/training.js index 18418388..c69b8ada 100644 --- a/server/src/services/roverLearningService/training.js +++ b/server/src/services/roverLearningService/training.js @@ -4,7 +4,7 @@ const { errorDetails } = require('./errors'); const fs = require('node:fs/promises'); const path = require('node:path'); const { spawn } = require('node:child_process'); -const { randomInt } = require('node:crypto'); +const { randomInt, createHash } = require('node:crypto'); const { adoptCheckpoint } = require('./models'); const { listRecordings } = require('./storage'); @@ -75,10 +75,27 @@ function createTrainer({ root, config, actionThreshold, logger }) { const other = randomInt(index + 1); [eligible[index], eligible[other]] = [eligible[other], eligible[index]]; } - const selected = eligible.slice(0, 4); + // Match workers/train.py exactly. Never move a previously trained session + // into evaluation just to fill a slot: that would contaminate the holdout. + const partitions = { training: [], heldOut: [] }; + for (const id of eligible) { + const heldOut = parseInt(createHash('sha256').update(id).digest('hex').slice(0, 8), 16) % 5 === 0; + partitions[heldOut ? 'heldOut' : 'training'].push(id); + } + const trainingSessions = partitions.training.slice(0, partitions.heldOut.length ? 3 : 4); + const heldOutSessions = trainingSessions.length ? partitions.heldOut.slice(0, 1) : []; + const selected = [...trainingSessions, ...heldOutSessions]; + const selection = { + atUnixMs: Date.now(), + availableTraining: partitions.training.length, availableHeldOut: partitions.heldOut.length, + selectedTraining: trainingSessions.length, selectedHeldOut: heldOutSessions.length, + reason: !trainingSessions.length ? 'Waiting for completed usable training-partition recordings' + : !heldOutSessions.length ? 'Training without evaluation: no eligible held-out recordings' : null, + }; + state = { ...state, selection }; nextRun = Date.now() + config.intervalSeconds * 1000; if (!selected.length) { - state = { ...state, status: 'waiting', reason: 'Waiting for completed usable human recordings' }; + state = { ...state, status: 'waiting', reason: selection.reason }; await saveLedger(); return; } @@ -86,7 +103,7 @@ function createTrainer({ root, config, actionThreshold, logger }) { await fs.writeFile(jobPath, JSON.stringify({ root, config, actionThreshold, sessions: selected })); if (stopped) return; selected.forEach((id) => leases.add(id)); - state = { ...state, status: 'training', reason: null, sessions: selected }; + state = { ...state, status: 'training', reason: selection.reason, sessions: selected }; state.startedAt = Date.now(); child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], { stdio: ['ignore', 'pipe', 'pipe'],