diff --git a/plans/rover learning service.md b/plans/rover learning service.md index cb0ee243..806770a8 100644 --- a/plans/rover learning service.md +++ b/plans/rover learning service.md @@ -9,8 +9,8 @@ network produces scores and numeric values; it never generates JSON text. Implemented: remove wheel-speed telemetry and previous drive-command history from inputs (raw recordings unchanged); preserve exact numeric zeros; reuse -recordings with a fresh resume-v4.pt / schedule-v4.json lineage. Existing format 2 -snapshots remain stored and are visibly incompatible. One ongoing learner, gated +recordings with a fresh resume-v4.pt / schedule-v4.json lineage. Startup deletes recognized older-format snapshots/checkpoints and old training +lineages; recordings and current-format models are preserved. One ongoing learner, gated Latest/Previous, and admin-created permanent snapshots remain. Main card shows decoded command parameters, score, applied threshold marker, and @@ -123,7 +123,7 @@ the default development path. Intended layout: rover-learning/ recordings/ Rolling video/events/session metadata (implemented) models/ Permanent UUID directories: weights.pt and model.json - training/ resume-v3.pt, schedule-v3.json, and current job.json + training/ resume-v4.pt, schedule-v4.json, and current job.json ``` Training/model directories are created only when service and training are enabled. @@ -914,3 +914,19 @@ temporal latching or accumulated motion from random reservoir samples. It does not claim to verify actual physical command execution. After normal deployment, let the new learner process retained recordings; copied reportVersion 3 diagnostics provide the next evidence. Do not demand prescribed driving or a manual checklist. + +### Obsolete model cleanup after format 3 transition + +Latest user IPI explicitly authorizes deleting obsolete snapshots, checkpoints and +training state. This supersedes earlier format 2 preservation notes above. +Enabled-service startup, before trainer/driving/gateway startup, scans both models/ +and training/checkpoints/. Identity-validated manifests with integer policy version +>=1 and < current POLICY_VERSION are removed. Current/future/unknown-format models +and recordings are preserved. Candidate/Latest/Previous references to obsolete +checkpoints are atomically removed before directory deletion; leased directories +are retired until release. Cleanup also removes legacy resume/schedule files and +temporary files for the unversioned and v1-v3 lineages. Current v4 state is kept. +Cleanup runs even when training is disabled, but not when the whole service is off. +Production deletion occurs on the next enabled-service startup after deployment. +Verification: focused Node syntax and diff whitespace checks passed. No production +storage was accessed; actual deletion awaits enabled-service startup on the server. diff --git a/server/src/services/roverLearningService/models.js b/server/src/services/roverLearningService/models.js index 646dc823..92e4d22f 100644 --- a/server/src/services/roverLearningService/models.js +++ b/server/src/services/roverLearningService/models.js @@ -20,7 +20,7 @@ async function index(root) { } async function metadata(directory) { const model = JSON.parse(await fs.readFile(path.join(directory, 'model.json'), 'utf8')); - if (model.id !== path.basename(directory)) throw new Error('Model identity mismatch'); + if (model?.id !== path.basename(directory)) throw new Error('Model identity mismatch'); return model; } async function prune(root, state) { @@ -100,22 +100,53 @@ function saveSnapshot(root) { return model; }); } -// Explicitly retire format 1 models after the wheel-policy replacement. Only -// recognized old-format manifests qualify; malformed or future models are kept. +// Retire recognized older policies before workers start. Unknown/malformed or +// future manifests are not evidence that a directory is disposable. function removeObsoleteModels(root) { return serial(async () => { - const directory = path.join(root, 'models'); - let entries; - try { entries = await fs.readdir(directory, { withFileTypes: true }); } - catch (error) { if (error.code === 'ENOENT') return; throw error; } - for (const entry of entries) { - if (!entry.isDirectory() || !UUID.test(entry.name)) continue; - const target = path.join(directory, entry.name); - let model; - try { model = await metadata(target); } - catch { continue; } - if (model.specification?.version === 1 && !leases.has(target)) { - await fs.rm(target, { recursive: true }); + const obsolete = []; + for (const directory of [path.join(root, 'models'), checkpointRoot(root)]) { + let entries; + try { entries = await fs.readdir(directory, { withFileTypes: true }); } + catch (error) { if (error.code === 'ENOENT') continue; throw error; } + for (const entry of entries) { + if (!entry.isDirectory() || !UUID.test(entry.name)) continue; + const target = path.join(directory, entry.name); + let model; + try { model = await metadata(target); } + catch (error) { + if (error.code === 'ENOENT' || error instanceof SyntaxError || error.message === 'Model identity mismatch') continue; + throw error; + } + const version = model.specification?.version; + if (Number.isInteger(version) && version >= 1 && version < POLICY_VERSION) { + obsolete.push({ target, checkpoint: directory === checkpointRoot(root), id: entry.name }); + } + } + } + const state = await index(root); + const oldCheckpoints = new Set(obsolete.filter((entry) => entry.checkpoint).map((entry) => entry.id)); + let changed = false; + for (const alias of ['candidate', 'latest', 'previous']) { + if (oldCheckpoints.has(state[alias])) { delete state[alias]; changed = true; } + } + // Remove aliases atomically first, so interruption cannot leave the catalog + // referring to a checkpoint that cleanup already deleted. + if (changed) { + const filename = path.join(checkpointRoot(root), 'index.json'); + await fs.writeFile(`${filename}.tmp`, JSON.stringify(state)); + await fs.rename(`${filename}.tmp`, filename); + } + for (const { target } of obsolete) { + if (leases.has(target)) retired.add(target); + else { await fs.rm(target, { recursive: true, force: true }); retired.delete(target); } + } + // These filenames belong to superseded optimizer/scheduler lineages, not + // policy versions. The current command policy uses lineage v4. + for (const suffix of ['', '-v1', '-v2', '-v3']) { + for (const filename of [`resume${suffix}.pt`, `resume${suffix}.tmp`, + `schedule${suffix}.json`, `schedule${suffix}.json.tmp`]) { + await fs.rm(path.join(root, 'training', filename), { force: true }); } } });