backup / restoreslop

server/src/configuration/database.js

@@ -372,6 +372,15 @@ function createConfigurationDatabase({ databasePath = DEFAULT_DATABASE_PATH } =
     return importConfigurationFileTransaction(payload);
   }
 
+  function backupDatabase(destinationPath) {
+    /*
+      SQLite's online backup API produces one coherent database file while the
+      live WAL-backed connection remains open. The backup service receives only
+      this narrow operation, never the private database handle.
+    */
+    return db.backup(destinationPath);
+  }
+
   return {
     databasePath,
     getActiveConfigurationRecord,
@@ -389,6 +398,7 @@ function createConfigurationDatabase({ databasePath = DEFAULT_DATABASE_PATH } =
     listAuditEvents,
     recordAuditEvent,
     importConfigurationFile,
+    backupDatabase,
     close: () => db.close(),
   };
 }

server/src/services/backupRestoreService/backup.js

@@ -0,0 +1,177 @@
+// Full Data Backup
+// Purpose: Creates one validated archive of the durable server data without stopping live services or writers.
+// Scope: Uses service-owned online database snapshots and stable file copies inside the canonical data directory.
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const crypto = require('crypto');
+const tar = require('tar');
+const Database = require('better-sqlite3');
+const { resolveDataDir, resolveDataPath } = require('../../helpers/dataPaths');
+const packageInfo = require('../../../package.json');
+
+const FORMAT_VERSION = 1;
+const CONTROL_DIR_NAME = 'backup-restore';
+const EXCLUDED_TOP_LEVEL_NAMES = new Set([CONTROL_DIR_NAME, 'runtime']);
+const DATABASE_NAMES = ['configuration.sqlite', 'identity.sqlite', 'fleet-reports.sqlite'];
+const DATABASE_FILES = new Set(DATABASE_NAMES.flatMap((name) => [name, `${name}-wal`, `${name}-shm`]));
+const FILE_COPY_ATTEMPTS = 3;
+
+function readDatabaseSchemaVersions(payloadDir) {
+  const configuration = new Database(path.join(payloadDir, 'configuration.sqlite'), { readonly: true });
+  const identity = new Database(path.join(payloadDir, 'identity.sqlite'), { readonly: true });
+  const fleetReports = new Database(path.join(payloadDir, 'fleet-reports.sqlite'), { readonly: true });
+  try {
+    return {
+      configuration: Number(configuration.prepare('SELECT MAX(version) AS version FROM schema_migrations').get()?.version) || 0,
+      identity: Number(identity.pragma('user_version', { simple: true })) || 0,
+      // Fleet reporting currently evolves with additive startup checks and has
+      // no numbered migration table, so zero accurately identifies that scheme.
+      fleetReports: Number(fleetReports.pragma('user_version', { simple: true })) || 0,
+    };
+  } finally {
+    configuration.close();
+    identity.close();
+    fleetReports.close();
+  }
+}
+
+function sha256File(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash('sha256');
+    const stream = fs.createReadStream(filePath);
+    stream.on('error', reject);
+    stream.on('data', (chunk) => hash.update(chunk));
+    stream.on('end', () => resolve(hash.digest('hex')));
+  });
+}
+
+async function copyStableFile(sourcePath, destinationPath) {
+  for (let attempt = 0; attempt < FILE_COPY_ATTEMPTS; attempt += 1) {
+    try {
+      const before = await fsp.stat(sourcePath);
+      if (!before.isFile()) throw new Error(`Backup source is not a regular file: ${sourcePath}`);
+      await fsp.mkdir(path.dirname(destinationPath), { recursive: true });
+      await fsp.copyFile(sourcePath, destinationPath);
+      await fsp.chmod(destinationPath, before.mode & 0o777);
+
+      /*
+        A writer may replace or append to a media file while it is copied. Size
+        and timestamp checks catch ordinary changes, while comparing hashes
+        catches a same-size replacement. Only the unstable file is retried;
+        no owning service is paused.
+      */
+      const [sourceHash, destinationHash] = await Promise.all([
+        sha256File(sourcePath),
+        sha256File(destinationPath),
+      ]);
+      // Read the final metadata only after both hashes finish. Starting this
+      // stat concurrently could miss a write that occurred during hashing.
+      const after = await fsp.stat(sourcePath);
+      if (before.size === after.size
+        && before.mtimeMs === after.mtimeMs
+        && sourceHash === destinationHash) return true;
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error;
+      // A rotating snapshot can disappear between directory enumeration and
+      // copying. Treat that exact race like any other unstable active file.
+    }
+    await fsp.rm(destinationPath, { force: true });
+  }
+  return false;
+}
+
+async function copyDurableTree(sourceDir, destinationDir, relativeDir = '') {
+  let entries;
+  try {
+    entries = await fsp.readdir(path.join(sourceDir, relativeDir), { withFileTypes: true });
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+
+  const skipped = [];
+  for (const entry of entries) {
+    const relativePath = path.join(relativeDir, entry.name);
+    if (!relativeDir && EXCLUDED_TOP_LEVEL_NAMES.has(entry.name)) continue;
+    if (!relativeDir && DATABASE_FILES.has(entry.name)) continue;
+    if (entry.isSymbolicLink()) throw new Error(`Backup cannot include symbolic link: ${relativePath}`);
+    if (entry.isDirectory()) {
+      skipped.push(...await copyDurableTree(sourceDir, destinationDir, relativePath));
+      continue;
+    }
+    if (!entry.isFile()) throw new Error(`Backup cannot include special file: ${relativePath}`);
+    const copied = await copyStableFile(
+      path.join(sourceDir, relativePath),
+      path.join(destinationDir, relativePath),
+    );
+    if (!copied) skipped.push(relativePath.split(path.sep).join('/'));
+  }
+  return skipped;
+}
+
+async function listManifestFiles(rootDir, relativeDir = '') {
+  const entries = await fsp.readdir(path.join(rootDir, relativeDir), { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const relativePath = path.join(relativeDir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await listManifestFiles(rootDir, relativePath));
+      continue;
+    }
+    if (!entry.isFile()) throw new Error(`Backup staging contains a special file: ${relativePath}`);
+    const filePath = path.join(rootDir, relativePath);
+    const stat = await fsp.stat(filePath);
+    files.push({
+      path: relativePath.split(path.sep).join('/'),
+      size: stat.size,
+      sha256: await sha256File(filePath),
+    });
+  }
+  return files.sort((left, right) => left.path.localeCompare(right.path));
+}
+
+async function createFullBackup({ configurationDatabase, identityService, fleetReportService, jobId }) {
+  const dataDir = resolveDataDir();
+  const jobDir = resolveDataPath(path.join(CONTROL_DIR_NAME, `backup-${jobId}`));
+  const payloadDir = path.join(jobDir, 'data');
+  const archivePath = path.join(jobDir, 'multirover-backup.tar.gz');
+  await fsp.rm(jobDir, { recursive: true, force: true });
+  await fsp.mkdir(payloadDir, { recursive: true });
+
+  try {
+    // Each database owner remains live and writes a coherent SQLite snapshot
+    // directly into the same staging tree as the ordinary durable files.
+    await Promise.all([
+      configurationDatabase.backupDatabase(path.join(payloadDir, DATABASE_NAMES[0])),
+      identityService.backupDatabase(path.join(payloadDir, DATABASE_NAMES[1])),
+      fleetReportService.backupDatabase(path.join(payloadDir, DATABASE_NAMES[2])),
+    ]);
+    const skippedUnstableFiles = await copyDurableTree(dataDir, payloadDir);
+    const files = await listManifestFiles(payloadDir);
+    const manifest = {
+      format: 'multirover-full-backup',
+      formatVersion: FORMAT_VERSION,
+      applicationVersion: packageInfo.version,
+      createdAt: Date.now(),
+      databaseSchemaVersions: readDatabaseSchemaVersions(payloadDir),
+      files,
+      skippedUnstableFiles,
+    };
+    await fsp.writeFile(path.join(jobDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
+    await tar.c({ cwd: jobDir, file: archivePath, gzip: true, portable: true }, ['manifest.json', 'data']);
+    return { archivePath, jobDir, manifest };
+  } catch (error) {
+    await fsp.rm(jobDir, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+module.exports = {
+  CONTROL_DIR_NAME,
+  DATABASE_NAMES,
+  FORMAT_VERSION,
+  createFullBackup,
+  readDatabaseSchemaVersions,
+  sha256File,
+};

server/src/services/backupRestoreService/backupRestore.test.js

@@ -0,0 +1,135 @@
+// Backup and Restore Service Tests
+// Purpose: Verifies complete archive round trips, exclusions, malicious entry rejection, startup replacement, and rollback.
+// Scope: Uses one isolated SERVER_DATA_DIR and injected SQLite owners; it never reads or changes development server data.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('fs');
+const fsp = require('fs/promises');
+const os = require('os');
+const path = require('path');
+const Database = require('better-sqlite3');
+const tar = require('tar');
+
+const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'multirover-backup-restore-'));
+process.env.SERVER_DATA_DIR = temporaryRoot;
+
+const { createFullBackup } = require('./backup');
+const { inspectArchive, prepareRestoreArchive, validateExtractedRestore } = require('./restore');
+const startupRestore = require('./startupRestore');
+const VALID_RESTORE_ID = 'a'.repeat(64);
+
+function createSourceDatabase(name) {
+  const filePath = path.join(temporaryRoot, `source-${name}`);
+  const database = new Database(filePath);
+  database.exec('CREATE TABLE example (value TEXT NOT NULL); INSERT INTO example VALUES (\'preserved\');');
+  if (name === 'configuration.sqlite') {
+    database.exec('CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY); INSERT INTO schema_migrations VALUES (1);');
+  } else if (name === 'identity.sqlite') {
+    database.pragma('user_version = 4');
+  }
+  return database;
+}
+
+test.after(() => {
+  fs.rmSync(temporaryRoot, { recursive: true, force: true });
+});
+
+test('creates and validates a complete backup while excluding runtime and control data', async () => {
+  await fsp.writeFile(path.join(temporaryRoot, 'state.json'), '{"preserved":true}\n');
+  await fsp.mkdir(path.join(temporaryRoot, 'replays'), { recursive: true });
+  await fsp.writeFile(path.join(temporaryRoot, 'replays', 'complete.mp4'), 'complete replay');
+  await fsp.mkdir(path.join(temporaryRoot, 'runtime'), { recursive: true });
+  await fsp.writeFile(path.join(temporaryRoot, 'runtime', 'active.tmp'), 'discard me');
+
+  const sources = ['configuration.sqlite', 'identity.sqlite', 'fleet-reports.sqlite']
+    .map(createSourceDatabase);
+  const owners = sources.map((database) => ({
+    backupDatabase: (destinationPath) => database.backup(destinationPath),
+  }));
+  const result = await createFullBackup({
+    configurationDatabase: owners[0],
+    identityService: owners[1],
+    fleetReportService: owners[2],
+    jobId: 'test-backup',
+  });
+
+  assert.ok(fs.statSync(result.archivePath).size > 0);
+  assert.ok(result.manifest.files.some((entry) => entry.path === 'state.json'));
+  assert.ok(result.manifest.files.some((entry) => entry.path === 'replays/complete.mp4'));
+  assert.equal(result.manifest.files.some((entry) => entry.path.includes('runtime')), false);
+  assert.equal(result.manifest.files.some((entry) => entry.path.includes('backup-restore')), false);
+
+  const restoreJob = path.join(temporaryRoot, 'backup-restore', `restore-${VALID_RESTORE_ID}`);
+  await fsp.mkdir(restoreJob, { recursive: true });
+  const uploadedArchive = path.join(restoreJob, 'upload.tar.gz');
+  await fsp.copyFile(result.archivePath, uploadedArchive);
+  const summary = await prepareRestoreArchive({ archivePath: uploadedArchive, jobDir: restoreJob, actor: 'test' });
+  assert.equal(summary.fileCount, result.manifest.files.length);
+
+  sources.forEach((database) => database.close());
+});
+
+test('rejects a staged restore whose contents no longer match the manifest', async () => {
+  const restoreJob = path.join(temporaryRoot, 'backup-restore', `restore-${VALID_RESTORE_ID}`);
+  const statePath = path.join(restoreJob, 'extracted', 'data', 'state.json');
+  await fsp.writeFile(statePath, '{"tampered":true}\n');
+  await assert.rejects(validateExtractedRestore(path.join(restoreJob, 'extracted')), /checksum or size mismatch/);
+  // Restore the known source content so the following startup-application test
+  // continues to exercise a genuinely validated replacement payload.
+  await fsp.writeFile(statePath, '{"preserved":true}\n');
+});
+
+test('rejects symbolic links before extracting an archive', async () => {
+  const unsafeRoot = path.join(temporaryRoot, 'unsafe-archive');
+  await fsp.mkdir(path.join(unsafeRoot, 'data'), { recursive: true });
+  await fsp.writeFile(path.join(unsafeRoot, 'manifest.json'), '{}');
+  await fsp.symlink('/etc/passwd', path.join(unsafeRoot, 'data', 'escape'));
+  const archivePath = path.join(temporaryRoot, 'unsafe.tar.gz');
+  await tar.c({ cwd: unsafeRoot, file: archivePath, gzip: true }, ['manifest.json', 'data']);
+  await assert.rejects(inspectArchive(archivePath), /unsupported entry type/);
+});
+
+test('applies validated replacement data and removes rollback only after startup succeeds', () => {
+  const restoreId = VALID_RESTORE_ID;
+  const jobDir = path.join(temporaryRoot, 'backup-restore', `restore-${restoreId}`);
+  fs.writeFileSync(path.join(temporaryRoot, 'old-state.txt'), 'old');
+  fs.mkdirSync(path.join(temporaryRoot, 'runtime'), { recursive: true });
+  fs.writeFileSync(path.join(temporaryRoot, 'runtime', 'active.tmp'), 'discard during restore');
+  startupRestore.writeJson(startupRestore.pendingPath, {
+    restoreId,
+    state: 'pending',
+    requestedAt: Date.now(),
+    actor: 'test',
+  });
+
+  const applied = startupRestore.applyPendingRestore();
+  assert.equal(applied.status, 'awaiting-health');
+  assert.equal(fs.existsSync(path.join(temporaryRoot, 'old-state.txt')), false);
+  assert.equal(fs.existsSync(path.join(temporaryRoot, 'runtime')), false);
+  assert.equal(fs.readFileSync(path.join(temporaryRoot, 'state.json'), 'utf8'), '{"preserved":true}\n');
+  assert.equal(fs.existsSync(path.join(temporaryRoot, 'backup-restore', 'rollback', 'old-state.txt')), true);
+
+  const completed = startupRestore.markStartupSuccessful();
+  assert.equal(completed.status, 'restored');
+  assert.equal(fs.existsSync(jobDir), false);
+  assert.equal(fs.existsSync(startupRestore.pendingPath), false);
+});
+
+test('restores the rollback copy when a replaced application did not reach health', () => {
+  const restoreId = 'b'.repeat(64);
+  const rollbackDir = path.join(temporaryRoot, 'backup-restore', 'rollback');
+  fs.mkdirSync(rollbackDir, { recursive: true });
+  fs.writeFileSync(path.join(rollbackDir, 'state.json'), 'previous state');
+  fs.writeFileSync(path.join(temporaryRoot, 'state.json'), 'failed replacement');
+  startupRestore.writeJson(startupRestore.pendingPath, {
+    restoreId,
+    state: 'awaiting-health',
+    requestedAt: Date.now(),
+    actor: 'test',
+  });
+
+  const result = startupRestore.applyPendingRestore();
+  assert.equal(result.status, 'rolled-back');
+  assert.equal(fs.readFileSync(path.join(temporaryRoot, 'state.json'), 'utf8'), 'previous state');
+  assert.equal(startupRestore.getLastRestoreResult().status, 'rolled-back');
+});

server/src/services/backupRestoreService/index.js

@@ -0,0 +1,267 @@
+// Backup and Restore Service
+// Purpose: Owns full-data archive creation, browser transfer, staged restore confirmation, startup replacement, and rollback.
+// Scope: All backup/restore orchestration stays in this service; database owners expose only their online snapshot operations.
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const crypto = require('crypto');
+const { Transform } = require('stream');
+const { pipeline } = require('stream/promises');
+const { resolveDataPath } = require('../../helpers/dataPaths');
+const { CONTROL_DIR_NAME, createFullBackup } = require('./backup');
+const { MAX_ARCHIVE_BYTES, prepareRestoreArchive } = require('./restore');
+const startupRestore = require('./startupRestore');
+
+const TOKEN_LIFETIME_MS = 10 * 60 * 1000;
+const controlDir = resolveDataPath(CONTROL_DIR_NAME);
+const downloads = new Map();
+const uploads = new Map();
+let backupInProgress = false;
+let registered = false;
+let app;
+let io;
+let logger;
+let getConfigurationDatabase;
+let identityService;
+let fleetReportService;
+let requireLockdownAdministrator;
+let requireRecentPassword;
+let requestApplicationRestart;
+let isApplicationRestartPending;
+
+function actorFor(socket) {
+  return socket?.data?.user?.username || socket.id;
+}
+
+function createToken() {
+  return crypto.randomBytes(32).toString('hex');
+}
+
+function pruneExpiredTransfers() {
+  const now = Date.now();
+  for (const [token, transfer] of [...downloads, ...uploads]) {
+    if (transfer.expiresAt > now) continue;
+    downloads.delete(token);
+    uploads.delete(token);
+    if (transfer.jobDir) fsp.rm(transfer.jobDir, { recursive: true, force: true }).catch(() => undefined);
+  }
+}
+
+function responseError(cb, error) {
+  logger.warn('Backup or restore request failed', { error: error.message });
+  cb({ error: error.message, code: error.code || null });
+}
+
+function removeOrphanedStaging() {
+  let pendingRestoreId = null;
+  try {
+    pendingRestoreId = JSON.parse(fs.readFileSync(startupRestore.pendingPath, 'utf8')).restoreId || null;
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error;
+  }
+  for (const entry of fs.readdirSync(controlDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const preservedName = pendingRestoreId ? `restore-${pendingRestoreId}` : null;
+    if ((entry.name.startsWith('backup-') || entry.name.startsWith('restore-'))
+      && entry.name !== preservedName) {
+      /*
+        Transfer authorization lives only in process memory. After a restart,
+        an unconfirmed upload or undownloaded archive can no longer be reached,
+        so deleting it prevents abandoned full-data copies from accumulating.
+      */
+      fs.rmSync(path.join(controlDir, entry.name), { recursive: true, force: true });
+    }
+  }
+}
+
+function registerSocketApi() {
+  io.on('connection', (socket) => {
+    socket.on('backupRestore:status', (_payload = {}, cb = () => {}) => {
+      try {
+        requireLockdownAdministrator(socket);
+        cb({ success: true, lastRestore: startupRestore.getLastRestoreResult() });
+      } catch (error) {
+        responseError(cb, error);
+      }
+    });
+
+    socket.on('backupRestore:createBackup', (_payload = {}, cb = () => {}) => {
+      Promise.resolve().then(async () => {
+        requireRecentPassword(socket);
+        if (backupInProgress) throw new Error('A full backup is already being created.');
+        backupInProgress = true;
+        try {
+          pruneExpiredTransfers();
+          const jobId = createToken();
+          const result = await createFullBackup({
+            configurationDatabase: getConfigurationDatabase(),
+            identityService,
+            fleetReportService,
+            jobId,
+          });
+          const token = createToken();
+          downloads.set(token, {
+            archivePath: result.archivePath,
+            jobDir: result.jobDir,
+            expiresAt: Date.now() + TOKEN_LIFETIME_MS,
+          });
+          getConfigurationDatabase().recordAuditEvent(actorFor(socket), 'backup.created', {
+            fileCount: result.manifest.files.length,
+            skippedUnstableFileCount: result.manifest.skippedUnstableFiles.length,
+          });
+          cb({
+            success: true,
+            downloadUrl: `/admin-api/backup/${token}`,
+            fileCount: result.manifest.files.length,
+            skippedUnstableFiles: result.manifest.skippedUnstableFiles,
+          });
+        } finally {
+          backupInProgress = false;
+        }
+      }).catch((error) => responseError(cb, error));
+    });
+
+    socket.on('backupRestore:createRestoreUpload', (_payload = {}, cb = () => {}) => {
+      try {
+        requireRecentPassword(socket);
+        pruneExpiredTransfers();
+        const token = createToken();
+        uploads.set(token, {
+          actor: actorFor(socket),
+          expiresAt: Date.now() + TOKEN_LIFETIME_MS,
+        });
+        cb({ success: true, uploadUrl: `/admin-api/restore/${token}` });
+      } catch (error) {
+        responseError(cb, error);
+      }
+    });
+
+    socket.on('backupRestore:confirmRestore', ({ restoreId } = {}, cb = () => {}) => {
+      try {
+        requireRecentPassword(socket);
+        const safeRestoreId = String(restoreId || '');
+        if (!/^[a-f0-9]{64}$/.test(safeRestoreId)) throw new Error('Validated restore was not found.');
+        if (isApplicationRestartPending()) throw new Error('Application restart already pending.');
+        if (fs.existsSync(startupRestore.pendingPath)) throw new Error('A restore is already pending.');
+        const jobDir = path.join(controlDir, `restore-${safeRestoreId}`);
+        if (!fs.existsSync(path.join(jobDir, 'validated.json'))) throw new Error('Validated restore was not found.');
+        startupRestore.writeJson(startupRestore.pendingPath, {
+          restoreId: safeRestoreId,
+          state: 'pending',
+          requestedAt: Date.now(),
+          actor: actorFor(socket),
+        });
+        getConfigurationDatabase().recordAuditEvent(actorFor(socket), 'restore.requested', { restoreId: safeRestoreId });
+        requestApplicationRestart({ actor: actorFor(socket), reason: 'restore-requested' });
+        cb({ success: true });
+      } catch (error) {
+        responseError(cb, error);
+      }
+    });
+  });
+}
+
+function registerHttpApi() {
+  app.get('/admin-api/backup/:token', (req, res) => {
+    pruneExpiredTransfers();
+    const transfer = downloads.get(String(req.params.token || ''));
+    if (!transfer) {
+      res.status(404).send('Backup download is missing or expired.');
+      return;
+    }
+    downloads.delete(req.params.token);
+    res.set({
+      'Content-Type': 'application/gzip',
+      'Content-Disposition': `attachment; filename="multirover-backup-${new Date().toISOString().slice(0, 10)}.tar.gz"`,
+      'Cache-Control': 'no-store',
+    });
+    const cleanup = () => fsp.rm(transfer.jobDir, { recursive: true, force: true }).catch(() => undefined);
+    res.once('close', cleanup);
+    fs.createReadStream(transfer.archivePath).on('error', (error) => {
+      logger.warn('Backup download failed', { error: error.message });
+      if (!res.headersSent) res.status(500).end();
+      else res.destroy(error);
+    }).pipe(res);
+  });
+
+  app.put('/admin-api/restore/:token', async (req, res) => {
+    pruneExpiredTransfers();
+    const token = String(req.params.token || '');
+    const transfer = uploads.get(token);
+    uploads.delete(token);
+    if (!transfer) {
+      res.status(404).json({ error: 'Restore upload is missing or expired.' });
+      return;
+    }
+    const contentLength = Number(req.headers['content-length']);
+    if (Number.isFinite(contentLength) && contentLength > MAX_ARCHIVE_BYTES) {
+      res.status(413).json({ error: 'Backup archive exceeds the restore size limit.' });
+      return;
+    }
+
+    const restoreId = createToken();
+    const jobDir = path.join(controlDir, `restore-${restoreId}`);
+    const archivePath = path.join(jobDir, 'upload.tar.gz');
+    try {
+      await fsp.mkdir(jobDir, { recursive: true });
+      let receivedBytes = 0;
+      const limiter = new Transform({
+        transform(chunk, _encoding, callback) {
+          receivedBytes += chunk.length;
+          callback(receivedBytes > MAX_ARCHIVE_BYTES
+            ? new Error('Backup archive exceeds the restore size limit.')
+            : null, chunk);
+        },
+      });
+      await pipeline(req, limiter, fs.createWriteStream(archivePath, { mode: 0o600 }));
+      const summary = await prepareRestoreArchive({ archivePath, jobDir, actor: transfer.actor });
+      getConfigurationDatabase().recordAuditEvent(transfer.actor, 'restore.validated', {
+        restoreId,
+        fileCount: summary.fileCount,
+        totalBytes: summary.totalBytes,
+      });
+      res.set('Cache-Control', 'no-store').json({ success: true, restoreId, summary });
+    } catch (error) {
+      await fsp.rm(jobDir, { recursive: true, force: true });
+      logger.warn('Restore upload failed validation', { actor: transfer.actor, error: error.message });
+      res.status(error.message.includes('size limit') ? 413 : 400).json({ error: error.message });
+    }
+  });
+}
+
+function register() {
+  if (registered) return;
+  registered = true;
+  /*
+    These runtime dependencies are deliberately loaded only after earliest
+    startup restore has run. Several of them open SQLite immediately, so
+    importing them at module scope would make replacement too late and unsafe.
+  */
+  ({ app } = require('../../globals/http'));
+  io = require('../../globals/io');
+  logger = require('../../globals/logger').child('backupRestoreService');
+  ({ getConfigurationDatabase } = require('../../configuration'));
+  identityService = require('../identityService');
+  fleetReportService = require('../fleetReportService');
+  ({ requireLockdownAdministrator, requireRecentPassword } = require('../adminConfigurationService'));
+  ({ isApplicationRestartPending, requestApplicationRestart } = require('../serverControlService'));
+  fs.mkdirSync(controlDir, { recursive: true });
+  removeOrphanedStaging();
+  registerHttpApi();
+  registerSocketApi();
+}
+
+function markStartupSuccessful() {
+  const result = startupRestore.markStartupSuccessful();
+  if (result) {
+    const configuration = require('../../configuration');
+    configuration.getConfigurationDatabase().recordAuditEvent('system', 'restore.completed', { restoreId: result.restoreId });
+  }
+  return result;
+}
+
+module.exports = {
+  applyPendingRestore: startupRestore.applyPendingRestore,
+  markStartupSuccessful,
+  register,
+};

server/src/services/backupRestoreService/restore.js

@@ -0,0 +1,194 @@
+// Full Data Restore Validation
+// Purpose: Safely extracts and validates an uploaded MultiRover backup before it can become a pending restore.
+// Scope: Never changes active data; startupRestore owns the later replacement and rollback transaction.
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const Database = require('better-sqlite3');
+const tar = require('tar');
+const {
+  DATABASE_NAMES,
+  FORMAT_VERSION,
+  readDatabaseSchemaVersions,
+  sha256File,
+} = require('./backup');
+
+const MAX_ARCHIVE_BYTES = 100 * 1024 * 1024 * 1024;
+const MAX_EXTRACTED_BYTES = 200 * 1024 * 1024 * 1024;
+const MAX_ARCHIVE_ENTRIES = 100000;
+const RESERVED_DATA_NAMES = new Set(['backup-restore', 'runtime']);
+const SUPPORTED_DATABASE_SCHEMA_VERSIONS = {
+  configuration: 1,
+  identity: 4,
+  fleetReports: 0,
+};
+
+function normalizeArchivePath(value) {
+  const raw = String(value || '');
+  if (!raw || raw.includes('\\') || path.posix.isAbsolute(raw)) return null;
+  const withoutTrailingSlash = raw.replace(/\/+$/, '');
+  const normalized = path.posix.normalize(withoutTrailingSlash);
+  if (!normalized || normalized === '.' || normalized === '..' || normalized.startsWith('../')) return null;
+  return normalized;
+}
+
+function assertAllowedArchiveEntry(entry) {
+  const normalized = normalizeArchivePath(entry.path);
+  if (!normalized || (normalized !== 'manifest.json' && normalized !== 'data' && !normalized.startsWith('data/'))) {
+    throw new Error(`Backup contains an invalid archive path: ${entry.path}`);
+  }
+  if (!['File', 'Directory'].includes(entry.type)) {
+    throw new Error(`Backup contains unsupported entry type ${entry.type}: ${entry.path}`);
+  }
+  if (normalized.startsWith('data/')) {
+    const topLevelName = normalized.slice('data/'.length).split('/')[0];
+    if (RESERVED_DATA_NAMES.has(topLevelName)) {
+      throw new Error(`Backup contains reserved data path: ${entry.path}`);
+    }
+  }
+  return normalized;
+}
+
+async function inspectArchive(archivePath) {
+  let entryCount = 0;
+  let extractedBytes = 0;
+  const paths = new Set();
+  let validationError = null;
+  await tar.t({
+    file: archivePath,
+    strict: true,
+    onentry: (entry) => {
+      if (validationError) return;
+      try {
+        entryCount += 1;
+        extractedBytes += Number(entry.size) || 0;
+        if (entryCount > MAX_ARCHIVE_ENTRIES) throw new Error('Backup contains too many files.');
+        if (extractedBytes > MAX_EXTRACTED_BYTES) throw new Error('Backup expands beyond the restore size limit.');
+        const normalized = assertAllowedArchiveEntry(entry);
+        if (paths.has(normalized)) throw new Error(`Backup contains duplicate path: ${normalized}`);
+        paths.add(normalized);
+      } catch (error) {
+        /*
+          tar invokes onentry from its parser event stack, where throwing would
+          become an uncaught exception instead of rejecting tar.t(). Retain the
+          first failure and raise it immediately after the bounded listing.
+        */
+        validationError = error;
+      }
+    },
+  });
+  if (validationError) throw validationError;
+  if (!paths.has('manifest.json') || !paths.has('data')) {
+    throw new Error('Backup must contain manifest.json and one data directory.');
+  }
+}
+
+async function listExtractedFiles(rootDir, relativeDir = '') {
+  const entries = await fsp.readdir(path.join(rootDir, relativeDir), { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const relativePath = path.join(relativeDir, entry.name);
+    const fullPath = path.join(rootDir, relativePath);
+    const stat = await fsp.lstat(fullPath);
+    if (stat.isSymbolicLink()) throw new Error(`Restored data contains symbolic link: ${relativePath}`);
+    if (stat.isDirectory()) {
+      files.push(...await listExtractedFiles(rootDir, relativePath));
+      continue;
+    }
+    if (!stat.isFile()) throw new Error(`Restored data contains special file: ${relativePath}`);
+    files.push({
+      path: relativePath.split(path.sep).join('/'),
+      size: stat.size,
+      sha256: await sha256File(fullPath),
+    });
+  }
+  return files.sort((left, right) => left.path.localeCompare(right.path));
+}
+
+function verifySqliteDatabase(filePath, name) {
+  const database = new Database(filePath, { readonly: true, fileMustExist: true });
+  try {
+    const result = database.pragma('quick_check', { simple: true });
+    if (result !== 'ok') throw new Error(`${name} failed SQLite integrity validation.`);
+  } finally {
+    database.close();
+  }
+}
+
+async function validateExtractedRestore(extractDir) {
+  const manifestPath = path.join(extractDir, 'manifest.json');
+  const payloadDir = path.join(extractDir, 'data');
+  const manifestStat = await fsp.stat(manifestPath);
+  if (manifestStat.size > 10 * 1024 * 1024) throw new Error('Backup manifest is unreasonably large.');
+  const manifest = JSON.parse(await fsp.readFile(manifestPath, 'utf8'));
+  if (manifest.format !== 'multirover-full-backup' || manifest.formatVersion !== FORMAT_VERSION) {
+    throw new Error('Backup format or version is not supported.');
+  }
+  if (!Array.isArray(manifest.files)) throw new Error('Backup manifest has no file inventory.');
+
+  const expected = [...manifest.files].sort((left, right) => String(left.path).localeCompare(String(right.path)));
+  const actual = await listExtractedFiles(payloadDir);
+  if (expected.length !== actual.length) throw new Error('Backup file inventory does not match the archive.');
+  for (let index = 0; index < expected.length; index += 1) {
+    const wanted = expected[index];
+    const found = actual[index];
+    if (wanted.path !== found.path || wanted.size !== found.size || wanted.sha256 !== found.sha256) {
+      throw new Error(`Backup checksum or size mismatch: ${wanted.path || found.path}`);
+    }
+  }
+
+  for (const databaseName of DATABASE_NAMES) {
+    if (!actual.some((entry) => entry.path === databaseName)) {
+      throw new Error(`Backup is missing required database: ${databaseName}`);
+    }
+    verifySqliteDatabase(path.join(payloadDir, databaseName), databaseName);
+  }
+  const actualSchemaVersions = readDatabaseSchemaVersions(payloadDir);
+  for (const [databaseName, version] of Object.entries(actualSchemaVersions)) {
+    // JSON object property order has no meaning. Compare each known database by
+    // name so an otherwise valid manifest is not rejected merely because a
+    // different JSON writer emitted its keys in another order.
+    if (Number(manifest.databaseSchemaVersions?.[databaseName]) !== version) {
+      throw new Error('Backup database schema versions do not match its manifest.');
+    }
+    if (version > SUPPORTED_DATABASE_SCHEMA_VERSIONS[databaseName]) {
+      throw new Error(`Backup ${databaseName} database is newer than this application supports.`);
+    }
+  }
+  return manifest;
+}
+
+async function prepareRestoreArchive({ archivePath, jobDir, actor }) {
+  const archiveStat = await fsp.stat(archivePath);
+  if (!archiveStat.isFile() || archiveStat.size <= 0 || archiveStat.size > MAX_ARCHIVE_BYTES) {
+    throw new Error('Backup archive is empty or exceeds the restore size limit.');
+  }
+  await inspectArchive(archivePath);
+  const extractDir = path.join(jobDir, 'extracted');
+  await fsp.rm(extractDir, { recursive: true, force: true });
+  await fsp.mkdir(extractDir, { recursive: true });
+  // Data files never need executable or set-id permissions from an uploaded
+  // archive. Let the server account's umask choose safe extraction modes.
+  await tar.x({ cwd: extractDir, file: archivePath, strict: true, preservePaths: false, noChmod: true });
+  const manifest = await validateExtractedRestore(extractDir);
+  const summary = {
+    createdAt: manifest.createdAt,
+    applicationVersion: manifest.applicationVersion,
+    fileCount: manifest.files.length,
+    totalBytes: manifest.files.reduce((total, entry) => total + Number(entry.size || 0), 0),
+    skippedUnstableFiles: Array.isArray(manifest.skippedUnstableFiles) ? manifest.skippedUnstableFiles : [],
+  };
+  await fsp.writeFile(
+    path.join(jobDir, 'validated.json'),
+    `${JSON.stringify({ actor, validatedAt: Date.now(), summary }, null, 2)}\n`,
+    { encoding: 'utf8', mode: 0o600 },
+  );
+  return summary;
+}
+
+module.exports = {
+  MAX_ARCHIVE_BYTES,
+  inspectArchive,
+  prepareRestoreArchive,
+  validateExtractedRestore,
+};

server/src/services/backupRestoreService/startupRestore.js

@@ -0,0 +1,143 @@
+// Startup Data Restore
+// Purpose: Applies a previously validated restore before any application database opens and rolls back an interrupted start.
+// Scope: Operates only on top-level entries inside SERVER_DATA_DIR while preserving backupRestoreService control state.
+const fs = require('fs');
+const path = require('path');
+const { resolveDataDir, resolveDataPath } = require('../../helpers/dataPaths');
+const { CONTROL_DIR_NAME } = require('./backup');
+
+const controlDir = resolveDataPath(CONTROL_DIR_NAME);
+const pendingPath = path.join(controlDir, 'pending.json');
+const rollbackDir = path.join(controlDir, 'rollback');
+const lastResultPath = path.join(controlDir, 'last-result.json');
+
+function writeJson(filePath, value) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  const temporaryPath = `${filePath}.tmp`;
+  fs.writeFileSync(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
+  fs.renameSync(temporaryPath, filePath);
+}
+
+function readPending() {
+  try {
+    return JSON.parse(fs.readFileSync(pendingPath, 'utf8'));
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+function listActiveDataEntries({ includeRuntime = true } = {}) {
+  fs.mkdirSync(resolveDataDir(), { recursive: true });
+  return fs.readdirSync(resolveDataDir()).filter((name) => (
+    name !== CONTROL_DIR_NAME && (includeRuntime || name !== 'runtime')
+  ));
+}
+
+function removeActiveData() {
+  for (const name of listActiveDataEntries()) {
+    fs.rmSync(path.join(resolveDataDir(), name), { recursive: true, force: true });
+  }
+}
+
+function moveChildren(sourceDir, destinationDir) {
+  fs.mkdirSync(destinationDir, { recursive: true });
+  for (const name of fs.readdirSync(sourceDir)) {
+    fs.renameSync(path.join(sourceDir, name), path.join(destinationDir, name));
+  }
+}
+
+function restoreRollback(pending, errorMessage) {
+  removeActiveData();
+  moveChildren(rollbackDir, resolveDataDir());
+  writeJson(lastResultPath, {
+    status: 'rolled-back',
+    restoreId: pending.restoreId,
+    completedAt: Date.now(),
+    error: errorMessage,
+  });
+  fs.rmSync(pendingPath, { force: true });
+}
+
+function applyPendingRestore() {
+  const pending = readPending();
+  if (!pending) return null;
+
+  /*
+    Reaching startup again in either state means the replacement process did
+    not reach the HTTP-listening success marker. The complete rollback copy was
+    created before active data was touched, so restoring it is deterministic.
+  */
+  if (pending.state === 'applying' || pending.state === 'awaiting-health') {
+    restoreRollback(pending, 'The restored application did not finish starting.');
+    return { status: 'rolled-back', restoreId: pending.restoreId };
+  }
+
+  const jobDir = path.join(controlDir, `restore-${pending.restoreId}`);
+  const replacementDir = path.join(jobDir, 'extracted', 'data');
+  if (!fs.existsSync(path.join(jobDir, 'validated.json'))
+    || !fs.existsSync(replacementDir)
+    || !fs.statSync(replacementDir).isDirectory()) {
+    fs.rmSync(pendingPath, { force: true });
+    writeJson(lastResultPath, {
+      status: 'failed',
+      restoreId: pending.restoreId,
+      completedAt: Date.now(),
+      error: 'Validated restore staging is missing.',
+    });
+    return { status: 'failed', restoreId: pending.restoreId };
+  }
+
+  try {
+    fs.rmSync(rollbackDir, { recursive: true, force: true });
+    fs.mkdirSync(rollbackDir, { recursive: true });
+    // Startup runs before any database or writer opens. Copying the entire
+    // current payload first gives every later replacement step one complete,
+    // local rollback source even if the process is interrupted halfway through.
+    for (const name of listActiveDataEntries({ includeRuntime: false })) {
+      fs.cpSync(path.join(resolveDataDir(), name), path.join(rollbackDir, name), { recursive: true });
+    }
+    writeJson(pendingPath, { ...pending, state: 'applying', applyingAt: Date.now() });
+    removeActiveData();
+    moveChildren(replacementDir, resolveDataDir());
+    writeJson(pendingPath, { ...pending, state: 'awaiting-health', appliedAt: Date.now() });
+    return { status: 'awaiting-health', restoreId: pending.restoreId };
+  } catch (error) {
+    if (fs.existsSync(rollbackDir)) restoreRollback(pending, error.message);
+    else fs.rmSync(pendingPath, { force: true });
+    return { status: 'rolled-back', restoreId: pending.restoreId, error: error.message };
+  }
+}
+
+function markStartupSuccessful() {
+  const pending = readPending();
+  if (!pending || pending.state !== 'awaiting-health') return null;
+  const jobDir = path.join(controlDir, `restore-${pending.restoreId}`);
+  fs.rmSync(rollbackDir, { recursive: true, force: true });
+  fs.rmSync(jobDir, { recursive: true, force: true });
+  fs.rmSync(pendingPath, { force: true });
+  const result = {
+    status: 'restored',
+    restoreId: pending.restoreId,
+    completedAt: Date.now(),
+  };
+  writeJson(lastResultPath, result);
+  return result;
+}
+
+function getLastRestoreResult() {
+  try {
+    return JSON.parse(fs.readFileSync(lastResultPath, 'utf8'));
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+module.exports = {
+  applyPendingRestore,
+  getLastRestoreResult,
+  markStartupSuccessful,
+  pendingPath,
+  writeJson,
+};

server/src/services/fleetReportService/index.js

@@ -136,5 +136,17 @@ module.exports = {
   get reportBuilder() {
     return runtime?.reportBuilder || null;
   },
+  backupDatabase(destinationPath) {
+    /*
+      Backups include reporting history even when collection is currently
+      disabled. Lazily opening the existing store keeps this one operation
+      behind the report service's normal database ownership boundary.
+    */
+    if (!storage) {
+      storage = createStorage({ logger });
+      storage.open();
+    }
+    return storage.backupDatabase(destinationPath);
+  },
   stop: stopRuntime,
 };

server/src/services/fleetReportService/storage.js

@@ -508,6 +508,16 @@ function createStorage({ logger }) {
     }), { available: false, path: DB_PATH });
   }
 
+  function backupDatabase(destinationPath) {
+    /*
+      Fleet collection may continue during an online SQLite backup. Each
+      resulting database file represents a valid point-in-time snapshot even
+      when new telemetry commits before the copy completes.
+    */
+    if (!open()) throw new Error('Fleet report database is unavailable.');
+    return db.backup(destinationPath);
+  }
+
   return {
     open,
     insertEvent,
@@ -525,6 +535,7 @@ function createStorage({ logger }) {
     getActiveBattery,
     replaceBattery,
     getDiagnostics,
+    backupDatabase,
   };
 }
 

server/src/services/httpServer/index.js

@@ -5,6 +5,7 @@ const { httpServer } = require('../../globals/http');
 const config = require('../../globals/config');
 const logger = require('../../globals/logger').child('httpServer');
 const { startMediaMtx } = require('../mediaMtxService');
+const backupRestoreService = require('../backupRestoreService');
 
 httpServer.listen(config.port, () => {
   logger.info(`Server listening on :${config.port}`);
@@ -14,6 +15,13 @@ httpServer.listen(config.port, () => {
     first publisher attempts to authenticate.
   */
   startMediaMtx();
+  /*
+    Give child processes and startup integrations a short stabilization window
+    after restored databases migrate and HTTP begins listening. If the process
+    exits during that window, earliest startup sees the awaiting-health marker
+    and restores the prior data instead of accepting a broken replacement.
+  */
+  setTimeout(() => backupRestoreService.markStartupSuccessful(), 5000);
 });
 
 function stopAcceptingConnections() {

server/src/services/identityService/index.js

@@ -118,6 +118,15 @@ function getDb() {
   return db;
 }
 
+function backupDatabase(destinationPath) {
+  /*
+    Keep identity writes live while SQLite copies a transactionally consistent
+    view into backup staging. Exposing the operation instead of the connection
+    preserves this service as the sole owner of identity.sqlite.
+  */
+  return getDb().backup(destinationPath);
+}
+
 function ensureSchema(conn) {
   conn.exec(`
     create table if not exists users (
@@ -1087,6 +1096,7 @@ function createJsonStore({ path: filePath, normalizeStoreShape, cloneStore, logg
 module.exports = {
   identityEvents,
   getDb,
+  backupDatabase,
   sanitizeNickname,
   normalizeCookieUserId,
   isValidCookieUserId,

server/src/services/serverControlService/index.js

@@ -34,21 +34,30 @@ function scheduleApplicationRestart() {
   }, 250);
 }
 
+function requestApplicationRestart({ actor, reason = 'administrator-requested' }) {
+  if (restartPending) throw new Error('Application restart already pending.');
+  database.recordAuditEvent(actor, 'application.restart-requested', { reason });
+  scheduleApplicationRestart();
+  logger.warn('Application restart requested', { actor, reason });
+  // Restore and ordinary admin restarts share this one browser contract, so
+  // clients can explain the disconnect without knowing which control invoked it.
+  io.emit('server:restarting', { reason });
+}
+
 io.on('connection', (socket) => {
   socket.on('server:restartApplication', (_payload = {}, cb = () => {}) => {
     try {
       requireRecentPassword(socket);
       const actor = actorFor(socket);
-      if (restartPending) throw new Error('Application restart already pending.');
-      database.recordAuditEvent(actor, 'application.restart-requested');
-      scheduleApplicationRestart();
-      logger.warn('Application restart requested', { actor });
+      requestApplicationRestart({ actor });
       cb({ success: true });
-      // Every connected browser receives one explicit reason for the upcoming
-      // disconnect instead of interpreting the brief outage as a network fault.
-      io.emit('server:restarting', { reason: 'administrator-requested' });
     } catch (error) {
       cb({ error: error.message, code: error.code || null });
     }
   });
 });
+
+module.exports = {
+  isApplicationRestartPending: () => restartPending,
+  requestApplicationRestart,
+};