mirror of
https://github.com/legop3/MultiRoombaRover.git
synced 2026-11-18 04:32:23 -05:00
@@ -821,3 +821,17 @@ evaluation/cache changes remain proposals, not part of this repair.
|
|||||||
stream is available here; the race is fixed by inspection but cannot yet be
|
stream is available here; the race is fixed by inspection but cannot yet be
|
||||||
confirmed as the cause of the earlier production events. Previously discarded
|
confirmed as the cause of the earlier production events. Previously discarded
|
||||||
error details cannot be reconstructed. No new tests or processes were created.
|
error details cannot be reconstructed. No new tests or processes were created.
|
||||||
|
|
||||||
|
### Deliberate held-out selection
|
||||||
|
|
||||||
|
Latest IPI authorizes the scheduler correction only. Randomly shuffled eligible
|
||||||
|
sessions are now partitioned with exactly the worker's SHA256 first-eight-hex
|
||||||
|
modulo-five rule. Jobs select up to three training sessions and one held-out
|
||||||
|
session when both partitions exist; otherwise up to four training sessions.
|
||||||
|
No worker starts with held-out sessions alone. Existing split assignments, model
|
||||||
|
weights, and training progress are unchanged. Diagnostics expose timestamped
|
||||||
|
available/selected partition counts and the reason when evaluation cannot be
|
||||||
|
scheduled. Selection guarantees a held-out recording, not usable evaluation
|
||||||
|
windows: decoding/vocabulary/sample checks still apply in the worker.
|
||||||
|
Node syntax checks and diff whitespace checks passed; actual job selection on
|
||||||
|
production recordings remains unverified. No new tests or background processes.
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ function report(state, controller) {
|
|||||||
sessions: state.recording.sessions.map((item) => ({ startedAtUnixMs: item.startedAt,
|
sessions: state.recording.sessions.map((item) => ({ startedAtUnixMs: item.startedAt,
|
||||||
status: item.video, droppedEvents: item.droppedEvents, error: redact(item.error), failureDetails: item.failureDetails || null, stderr: redact(item.stderr), errorCategory: errorCategory(item.error) })) },
|
status: item.video, droppedEvents: item.droppedEvents, error: redact(item.error), failureDetails: item.failureDetails || null, stderr: redact(item.stderr), errorCategory: errorCategory(item.error) })) },
|
||||||
training: { status: training.status, steps: training.steps, examples: training.examples,
|
training: { status: training.status, steps: training.steps, examples: training.examples,
|
||||||
|
selection: training.selection || null,
|
||||||
loss: training.loss, losses: training.losses || null, publication: training.publication || null,
|
loss: training.loss, losses: training.losses || null, publication: training.publication || null,
|
||||||
evaluation: training.evaluation || null, dataset: metrics.dataset || training.dataset || null,
|
evaluation: training.evaluation || null, dataset: metrics.dataset || training.dataset || null,
|
||||||
resources: { decodeSeconds: metrics.decodeSeconds ?? null, trainingSeconds: metrics.trainingSeconds ?? null,
|
resources: { decodeSeconds: metrics.decodeSeconds ?? null, trainingSeconds: metrics.trainingSeconds ?? null,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ const { errorDetails } = require('./errors');
|
|||||||
const fs = require('node:fs/promises');
|
const fs = require('node:fs/promises');
|
||||||
const path = require('node:path');
|
const path = require('node:path');
|
||||||
const { spawn } = require('node:child_process');
|
const { spawn } = require('node:child_process');
|
||||||
const { randomInt } = require('node:crypto');
|
const { randomInt, createHash } = require('node:crypto');
|
||||||
const { adoptCheckpoint } = require('./models');
|
const { adoptCheckpoint } = require('./models');
|
||||||
const { listRecordings } = require('./storage');
|
const { listRecordings } = require('./storage');
|
||||||
|
|
||||||
@@ -75,10 +75,27 @@ function createTrainer({ root, config, actionThreshold, logger }) {
|
|||||||
const other = randomInt(index + 1);
|
const other = randomInt(index + 1);
|
||||||
[eligible[index], eligible[other]] = [eligible[other], eligible[index]];
|
[eligible[index], eligible[other]] = [eligible[other], eligible[index]];
|
||||||
}
|
}
|
||||||
const selected = eligible.slice(0, 4);
|
// Match workers/train.py exactly. Never move a previously trained session
|
||||||
|
// into evaluation just to fill a slot: that would contaminate the holdout.
|
||||||
|
const partitions = { training: [], heldOut: [] };
|
||||||
|
for (const id of eligible) {
|
||||||
|
const heldOut = parseInt(createHash('sha256').update(id).digest('hex').slice(0, 8), 16) % 5 === 0;
|
||||||
|
partitions[heldOut ? 'heldOut' : 'training'].push(id);
|
||||||
|
}
|
||||||
|
const trainingSessions = partitions.training.slice(0, partitions.heldOut.length ? 3 : 4);
|
||||||
|
const heldOutSessions = trainingSessions.length ? partitions.heldOut.slice(0, 1) : [];
|
||||||
|
const selected = [...trainingSessions, ...heldOutSessions];
|
||||||
|
const selection = {
|
||||||
|
atUnixMs: Date.now(),
|
||||||
|
availableTraining: partitions.training.length, availableHeldOut: partitions.heldOut.length,
|
||||||
|
selectedTraining: trainingSessions.length, selectedHeldOut: heldOutSessions.length,
|
||||||
|
reason: !trainingSessions.length ? 'Waiting for completed usable training-partition recordings'
|
||||||
|
: !heldOutSessions.length ? 'Training without evaluation: no eligible held-out recordings' : null,
|
||||||
|
};
|
||||||
|
state = { ...state, selection };
|
||||||
nextRun = Date.now() + config.intervalSeconds * 1000;
|
nextRun = Date.now() + config.intervalSeconds * 1000;
|
||||||
if (!selected.length) {
|
if (!selected.length) {
|
||||||
state = { ...state, status: 'waiting', reason: 'Waiting for completed usable human recordings' };
|
state = { ...state, status: 'waiting', reason: selection.reason };
|
||||||
await saveLedger();
|
await saveLedger();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -86,7 +103,7 @@ function createTrainer({ root, config, actionThreshold, logger }) {
|
|||||||
await fs.writeFile(jobPath, JSON.stringify({ root, config, actionThreshold, sessions: selected }));
|
await fs.writeFile(jobPath, JSON.stringify({ root, config, actionThreshold, sessions: selected }));
|
||||||
if (stopped) return;
|
if (stopped) return;
|
||||||
selected.forEach((id) => leases.add(id));
|
selected.forEach((id) => leases.add(id));
|
||||||
state = { ...state, status: 'training', reason: null, sessions: selected };
|
state = { ...state, status: 'training', reason: selection.reason, sessions: selected };
|
||||||
state.startedAt = Date.now();
|
state.startedAt = Date.now();
|
||||||
child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], {
|
child = spawn(config.python, [path.join(__dirname, 'workers/train.py'), '--job', jobPath], {
|
||||||
stdio: ['ignore', 'pipe', 'pipe'],
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
|||||||
Reference in New Issue
Block a user