backup restore slopfix

docs/server-admin-container-migration.md

@@ -459,11 +459,12 @@ Implemented on 2026-09-14:
 - Replaced the privileged host-reboot action with one lockdown-only, recently confirmed, audited application restart on the admin Overview. Node announces the restart, stops accepting new HTTP connections, and signals itself after acknowledging the browser; the existing service signal hooks clean up owned child processes, and systemd now restarts clean application exits without making `systemctl stop` ineffective.
 - Added one protected backup-and-restore service and admin page. Backups keep the application online, use SQLite's online snapshot API for all three databases, make verified stable copies of the remaining durable files, and produce a checksummed archive through a short-lived one-use download. Restore uploads are size-limited, reject unsafe archive entries, verify the complete manifest, checksums, SQLite integrity, and supported schema versions, then remain staged until explicit recent-password confirmation.
 - Restore now uses the normal application restart rather than stopping services itself. The earliest server startup swaps the validated replacement into the data directory, retains one rollback copy, and removes that copy only after the restored application reaches a stabilization point; an interrupted or failed first startup automatically puts the previous data back on the following start. Backup/restore control files and all staging remain inside `data/backup-restore`.
+- Fixed production WAL-mode snapshots creating unmanifested SQLite `-wal` and `-shm` files during schema inspection. Backup and restore validation now remove only those temporary staged sidecars before archiving or applying data, and the regression fixture uses WAL mode to match the real databases.
 
 Local verification completed:
 
 - All 114 server tests passed, including populated legacy-style default coverage, complete schema-description and input-example coverage, file-backed setup-code lifecycle and symlink rejection, service-definition-derived feature projection, schema-derived secret paths, configuration defaults and strict validation, full-document revision conflicts, secret preservation, administrator invariants, setup and initialized-server YAML import safety, recursive removal of nonexistent fields, and the earlier filesystem coverage.
-- All 25 server test files passed after live application and backup/restore were added. The isolated backup/restore tests cover complete archive round trips, excluded runtime/control data, checksum tampering, unsafe symbolic-link entries, earliest-startup replacement, successful cleanup, and automatic rollback. Application-restart syntax, authorization wiring, and supervisor configuration were checked without exercising the real process signal on the development machine.
+- All 25 server test files passed after live application and backup/restore were added. The isolated backup/restore tests cover production-style WAL snapshots without transient sidecars, complete archive round trips, excluded runtime/control data, checksum tampering, unsafe symbolic-link entries, earliest-startup replacement, successful cleanup, and automatic rollback. Application-restart syntax, authorization wiring, and supervisor configuration were checked without exercising the real process signal on the development machine.
 - Focused admin, route, and identity UI lint passed.
 - All 20 existing focused web UI tests passed.
 - The production web UI build completed successfully and regenerated the checked-in server assets.

server/src/services/backupRestoreService/backup.js

@@ -17,6 +17,20 @@ const DATABASE_NAMES = ['configuration.sqlite', 'identity.sqlite', 'fleet-report
 const DATABASE_FILES = new Set(DATABASE_NAMES.flatMap((name) => [name, `${name}-wal`, `${name}-shm`]));
 const FILE_COPY_ATTEMPTS = 3;
 
+async function removeSnapshotSidecars(payloadDir) {
+  /*
+    SQLite online backup produces a complete standalone main database file.
+    Reopening that snapshot to inspect its schema can still create empty WAL
+    and shared-memory coordination files because the database retains WAL as
+    its journal mode. Those files describe no durable backup content and must
+    be removed before the manifest inventory and tar archive are produced.
+  */
+  await Promise.all(DATABASE_NAMES.flatMap((name) => [
+    fsp.rm(path.join(payloadDir, `${name}-wal`), { force: true }),
+    fsp.rm(path.join(payloadDir, `${name}-shm`), { force: true }),
+  ]));
+}
+
 function readDatabaseSchemaVersions(payloadDir) {
   const configuration = new Database(path.join(payloadDir, 'configuration.sqlite'), { readonly: true });
   const identity = new Database(path.join(payloadDir, 'identity.sqlite'), { readonly: true });
@@ -148,13 +162,20 @@ async function createFullBackup({ configurationDatabase, identityService, fleetR
       fleetReportService.backupDatabase(path.join(payloadDir, DATABASE_NAMES[2])),
     ]);
     const skippedUnstableFiles = await copyDurableTree(dataDir, payloadDir);
+    /*
+      Finish every operation that can create a staged file before inventorying
+      the payload. Production databases use WAL mode, so schema inspection must
+      precede both sidecar cleanup and the final immutable file list.
+    */
+    const databaseSchemaVersions = readDatabaseSchemaVersions(payloadDir);
+    await removeSnapshotSidecars(payloadDir);
     const files = await listManifestFiles(payloadDir);
     const manifest = {
       format: 'multirover-full-backup',
       formatVersion: FORMAT_VERSION,
       applicationVersion: packageInfo.version,
       createdAt: Date.now(),
-      databaseSchemaVersions: readDatabaseSchemaVersions(payloadDir),
+      databaseSchemaVersions,
       files,
       skippedUnstableFiles,
     };
@@ -173,5 +194,6 @@ module.exports = {
   FORMAT_VERSION,
   createFullBackup,
   readDatabaseSchemaVersions,
+  removeSnapshotSidecars,
   sha256File,
 };

server/src/services/backupRestoreService/backupRestore.test.js

@@ -19,8 +19,16 @@ const startupRestore = require('./startupRestore');
 const VALID_RESTORE_ID = 'a'.repeat(64);
 
 function createSourceDatabase(name) {
-  const filePath = path.join(temporaryRoot, `source-${name}`);
+  /*
+    Real service databases use WAL mode. Keeping fixture sources under the
+    excluded runtime directory both mirrors that behavior and ensures their
+    own live WAL files are not mistaken for ordinary durable backup content.
+  */
+  const sourceDirectory = path.join(temporaryRoot, 'runtime', 'database-sources');
+  fs.mkdirSync(sourceDirectory, { recursive: true });
+  const filePath = path.join(sourceDirectory, name);
   const database = new Database(filePath);
+  database.pragma('journal_mode = WAL');
   database.exec('CREATE TABLE example (value TEXT NOT NULL); INSERT INTO example VALUES (\'preserved\');');
   if (name === 'configuration.sqlite') {
     database.exec('CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY); INSERT INTO schema_migrations VALUES (1);');
@@ -58,6 +66,7 @@ test('creates and validates a complete backup while excluding runtime and contro
   assert.ok(result.manifest.files.some((entry) => entry.path === 'replays/complete.mp4'));
   assert.equal(result.manifest.files.some((entry) => entry.path.includes('runtime')), false);
   assert.equal(result.manifest.files.some((entry) => entry.path.includes('backup-restore')), false);
+  assert.equal(result.manifest.files.some((entry) => entry.path.endsWith('-wal') || entry.path.endsWith('-shm')), false);
 
   const restoreJob = path.join(temporaryRoot, 'backup-restore', `restore-${VALID_RESTORE_ID}`);
   await fsp.mkdir(restoreJob, { recursive: true });
@@ -65,6 +74,8 @@ test('creates and validates a complete backup while excluding runtime and contro
   await fsp.copyFile(result.archivePath, uploadedArchive);
   const summary = await prepareRestoreArchive({ archivePath: uploadedArchive, jobDir: restoreJob, actor: 'test' });
   assert.equal(summary.fileCount, result.manifest.files.length);
+  const restoredNames = await fsp.readdir(path.join(restoreJob, 'extracted', 'data'));
+  assert.equal(restoredNames.some((name) => name.endsWith('-wal') || name.endsWith('-shm')), false);
 
   sources.forEach((database) => database.close());
 });

server/src/services/backupRestoreService/restore.js

@@ -10,6 +10,7 @@ const {
   DATABASE_NAMES,
   FORMAT_VERSION,
   readDatabaseSchemaVersions,
+  removeSnapshotSidecars,
   sha256File,
 } = require('./backup');
 
@@ -128,7 +129,22 @@ async function validateExtractedRestore(extractDir) {
 
   const expected = [...manifest.files].sort((left, right) => String(left.path).localeCompare(String(right.path)));
   const actual = await listExtractedFiles(payloadDir);
-  if (expected.length !== actual.length) throw new Error('Backup file inventory does not match the archive.');
+  if (expected.length !== actual.length) {
+    /*
+      Keep the strict complete-inventory check, but identify a few differences
+      so an operator can distinguish a missing file from an unexpected archive
+      entry without weakening restore validation or exposing file contents.
+    */
+    const expectedPaths = new Set(expected.map((entry) => entry.path));
+    const actualPaths = new Set(actual.map((entry) => entry.path));
+    const missing = expected.filter((entry) => !actualPaths.has(entry.path)).map((entry) => entry.path).slice(0, 5);
+    const unexpected = actual.filter((entry) => !expectedPaths.has(entry.path)).map((entry) => entry.path).slice(0, 5);
+    const details = [
+      missing.length ? `missing: ${missing.join(', ')}` : '',
+      unexpected.length ? `unexpected: ${unexpected.join(', ')}` : '',
+    ].filter(Boolean).join('; ');
+    throw new Error(`Backup file inventory does not match the archive${details ? ` (${details})` : ''}.`);
+  }
   for (let index = 0; index < expected.length; index += 1) {
     const wanted = expected[index];
     const found = actual[index];
@@ -155,6 +171,12 @@ async function validateExtractedRestore(extractDir) {
       throw new Error(`Backup ${databaseName} database is newer than this application supports.`);
     }
   }
+  /*
+    Integrity and schema reads can create fresh WAL coordination files even
+    though the uploaded snapshot initially matched its manifest exactly. Remove
+    those validation-only files so startup applies only inventoried content.
+  */
+  await removeSnapshotSidecars(payloadDir);
   return manifest;
 }