slop

plans/rover learning service.md

@@ -9,8 +9,8 @@ network produces scores and numeric values; it never generates JSON text.
 
 Implemented: remove wheel-speed telemetry and previous drive-command history
 from inputs (raw recordings unchanged); preserve exact numeric zeros; reuse
-recordings with a fresh resume-v4.pt / schedule-v4.json lineage. Existing format 2
-snapshots remain stored and are visibly incompatible. One ongoing learner, gated
+recordings with a fresh resume-v4.pt / schedule-v4.json lineage. Startup deletes recognized older-format snapshots/checkpoints and old training
+lineages; recordings and current-format models are preserved. One ongoing learner, gated
 Latest/Previous, and admin-created permanent snapshots remain.
 
 Main card shows decoded command parameters, score, applied threshold marker, and
@@ -123,7 +123,7 @@ the default development path. Intended layout:
 rover-learning/
   recordings/       Rolling video/events/session metadata (implemented)
   models/           Permanent UUID directories: weights.pt and model.json
-  training/         resume-v3.pt, schedule-v3.json, and current job.json
+  training/         resume-v4.pt, schedule-v4.json, and current job.json
 ```
 
 Training/model directories are created only when service and training are enabled.
@@ -914,3 +914,19 @@ temporal latching or accumulated motion from random reservoir samples. It does
 not claim to verify actual physical command execution. After normal deployment,
 let the new learner process retained recordings; copied reportVersion 3 diagnostics
 provide the next evidence. Do not demand prescribed driving or a manual checklist.
+
+### Obsolete model cleanup after format 3 transition
+
+Latest user IPI explicitly authorizes deleting obsolete snapshots, checkpoints and
+training state. This supersedes earlier format 2 preservation notes above.
+Enabled-service startup, before trainer/driving/gateway startup, scans both models/
+and training/checkpoints/. Identity-validated manifests with integer policy version
+>=1 and < current POLICY_VERSION are removed. Current/future/unknown-format models
+and recordings are preserved. Candidate/Latest/Previous references to obsolete
+checkpoints are atomically removed before directory deletion; leased directories
+are retired until release. Cleanup also removes legacy resume/schedule files and
+temporary files for the unversioned and v1-v3 lineages. Current v4 state is kept.
+Cleanup runs even when training is disabled, but not when the whole service is off.
+Production deletion occurs on the next enabled-service startup after deployment.
+Verification: focused Node syntax and diff whitespace checks passed. No production
+storage was accessed; actual deletion awaits enabled-service startup on the server.

server/src/services/roverLearningService/models.js

@@ -20,7 +20,7 @@ async function index(root) {
 }
 async function metadata(directory) {
   const model = JSON.parse(await fs.readFile(path.join(directory, 'model.json'), 'utf8'));
-  if (model.id !== path.basename(directory)) throw new Error('Model identity mismatch');
+  if (model?.id !== path.basename(directory)) throw new Error('Model identity mismatch');
   return model;
 }
 async function prune(root, state) {
@@ -100,22 +100,53 @@ function saveSnapshot(root) {
     return model;
   });
 }
-// Explicitly retire format 1 models after the wheel-policy replacement. Only
-// recognized old-format manifests qualify; malformed or future models are kept.
+// Retire recognized older policies before workers start. Unknown/malformed or
+// future manifests are not evidence that a directory is disposable.
 function removeObsoleteModels(root) {
   return serial(async () => {
-    const directory = path.join(root, 'models');
-    let entries;
-    try { entries = await fs.readdir(directory, { withFileTypes: true }); }
-    catch (error) { if (error.code === 'ENOENT') return; throw error; }
-    for (const entry of entries) {
-      if (!entry.isDirectory() || !UUID.test(entry.name)) continue;
-      const target = path.join(directory, entry.name);
-      let model;
-      try { model = await metadata(target); }
-      catch { continue; }
-      if (model.specification?.version === 1 && !leases.has(target)) {
-        await fs.rm(target, { recursive: true });
+    const obsolete = [];
+    for (const directory of [path.join(root, 'models'), checkpointRoot(root)]) {
+      let entries;
+      try { entries = await fs.readdir(directory, { withFileTypes: true }); }
+      catch (error) { if (error.code === 'ENOENT') continue; throw error; }
+      for (const entry of entries) {
+        if (!entry.isDirectory() || !UUID.test(entry.name)) continue;
+        const target = path.join(directory, entry.name);
+        let model;
+        try { model = await metadata(target); }
+        catch (error) {
+          if (error.code === 'ENOENT' || error instanceof SyntaxError || error.message === 'Model identity mismatch') continue;
+          throw error;
+        }
+        const version = model.specification?.version;
+        if (Number.isInteger(version) && version >= 1 && version < POLICY_VERSION) {
+          obsolete.push({ target, checkpoint: directory === checkpointRoot(root), id: entry.name });
+        }
+      }
+    }
+    const state = await index(root);
+    const oldCheckpoints = new Set(obsolete.filter((entry) => entry.checkpoint).map((entry) => entry.id));
+    let changed = false;
+    for (const alias of ['candidate', 'latest', 'previous']) {
+      if (oldCheckpoints.has(state[alias])) { delete state[alias]; changed = true; }
+    }
+    // Remove aliases atomically first, so interruption cannot leave the catalog
+    // referring to a checkpoint that cleanup already deleted.
+    if (changed) {
+      const filename = path.join(checkpointRoot(root), 'index.json');
+      await fs.writeFile(`${filename}.tmp`, JSON.stringify(state));
+      await fs.rename(`${filename}.tmp`, filename);
+    }
+    for (const { target } of obsolete) {
+      if (leases.has(target)) retired.add(target);
+      else { await fs.rm(target, { recursive: true, force: true }); retired.delete(target); }
+    }
+    // These filenames belong to superseded optimizer/scheduler lineages, not
+    // policy versions. The current command policy uses lineage v4.
+    for (const suffix of ['', '-v1', '-v2', '-v3']) {
+      for (const filename of [`resume${suffix}.pt`, `resume${suffix}.tmp`,
+        `schedule${suffix}.json`, `schedule${suffix}.json.tmp`]) {
+        await fs.rm(path.join(root, 'training', filename), { force: true });
       }
     }
   });