slop

plans/rover learning service.md

@@ -821,3 +821,17 @@ evaluation/cache changes remain proposals, not part of this repair.
   stream is available here; the race is fixed by inspection but cannot yet be
   confirmed as the cause of the earlier production events. Previously discarded
   error details cannot be reconstructed. No new tests or processes were created.
+
+### Deliberate held-out selection
+
+Latest IPI authorizes the scheduler correction only. Randomly shuffled eligible
+sessions are now partitioned with exactly the worker's SHA256 first-eight-hex
+modulo-five rule. Jobs select up to three training sessions and one held-out
+session when both partitions exist; otherwise up to four training sessions.
+No worker starts with held-out sessions alone. Existing split assignments, model
+weights, and training progress are unchanged. Diagnostics expose timestamped
+available/selected partition counts and the reason when evaluation cannot be
+scheduled. Selection guarantees a held-out recording, not usable evaluation
+windows: decoding/vocabulary/sample checks still apply in the worker.
+Node syntax checks and diff whitespace checks passed; actual job selection on
+production recordings remains unverified. No new tests or background processes.

server/src/services/roverLearningService/diagnostics.js

@@ -40,6 +40,7 @@ function report(state, controller) {
       sessions: state.recording.sessions.map((item) => ({ startedAtUnixMs: item.startedAt,
         status: item.video, droppedEvents: item.droppedEvents, error: redact(item.error), failureDetails: item.failureDetails || null, stderr: redact(item.stderr), errorCategory: errorCategory(item.error) })) },
     training: { status: training.status, steps: training.steps, examples: training.examples,
+      selection: training.selection || null,
       loss: training.loss, losses: training.losses || null, publication: training.publication || null,
       evaluation: training.evaluation || null, dataset: metrics.dataset || training.dataset || null,
       resources: { decodeSeconds: metrics.decodeSeconds ?? null, trainingSeconds: metrics.trainingSeconds ?? null,

server/src/services/roverLearningService/training.js

@@ -4,7 +4,7 @@ const { errorDetails } = require('./errors');
 const fs = require('node:fs/promises');
 const path = require('node:path');
 const { spawn } = require('node:child_process');
-const { randomInt } = require('node:crypto');
+const { randomInt, createHash } = require('node:crypto');
 const { adoptCheckpoint } = require('./models');
 const { listRecordings } = require('./storage');
 
@@ -75,10 +75,27 @@ function createTrainer({ root, config, actionThreshold, logger }) {
       const other = randomInt(index + 1);
       [eligible[index], eligible[other]] = [eligible[other], eligible[index]];
     }
-    const selected = eligible.slice(0, 4);
+    // Match workers/train.py exactly. Never move a previously trained session
+    // into evaluation just to fill a slot: that would contaminate the holdout.
+    const partitions = { training: [], heldOut: [] };
+    for (const id of eligible) {
+      const heldOut = parseInt(createHash('sha256').update(id).digest('hex').slice(0, 8), 16) % 5 === 0;
+      partitions[heldOut ? 'heldOut' : 'training'].push(id);
+    }
+    const trainingSessions = partitions.training.slice(0, partitions.heldOut.length ? 3 : 4);
+    const heldOutSessions = trainingSessions.length ? partitions.heldOut.slice(0, 1) : [];
+    const selected = [...trainingSessions, ...heldOutSessions];
+    const selection = {
+      atUnixMs: Date.now(),
+      availableTraining: partitions.training.length, availableHeldOut: partitions.heldOut.length,
+      selectedTraining: trainingSessions.length, selectedHeldOut: heldOutSessions.length,
+      reason: !trainingSessions.length ? 'Waiting for completed usable training-partition recordings'
+        : !heldOutSessions.length ? 'Training without evaluation: no eligible held-out recordings' : null,
+    };
+    state = { ...state, selection };
     nextRun = Date.now() + config.intervalSeconds * 1000;
     if (!selected.length) {
-      state = { ...state, status: 'waiting', reason: 'Waiting for completed usable human recordings' };
+      state = { ...state, status: 'waiting', reason: selection.reason };
       await saveLedger();
       return;
     }
@@ -86,7 +103,7 @@ function createTrainer({ root, config, actionThreshold, logger }) {
     await fs.writeFile(jobPath, JSON.stringify({ root, config, actionThreshold, sessions: selected }));
     if (stopped) return;
     selected.forEach((id) => leases.add(id));
-    state = { ...state, status: 'training', reason: null, sessions: selected };
+    state = { ...state, status: 'training', reason: selection.reason, sessions: selected };
     state.startedAt = Date.now();
     child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], {
       stdio: ['ignore', 'pipe', 'pipe'],