Files
MultiRoombaRover/server/src/services/serverControlService/controller.js
T
legop3 fde2fca0a0
Container image / image (push) Failing after 50s
add image tag selection to admin ui
2026-10-04 13:29:46 -04:00

381 lines
14 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Restricted Container Lifecycle Controller
// Purpose: Pulls and replaces only the fixed MultiRover application container through Docker's local API.
// Scope: Runs as the private lifecycle Compose service; it exposes no TCP listener and accepts tags only within the deployment-configured repository.
const fs = require('fs');
const path = require('path');
const http = require('http');
const Docker = require('dockerode');
const docker = new Docker({ socketPath: '/var/run/docker.sock' });
const TARGET_CONTAINER_NAME = 'multirover';
const TARGET_IMAGE = process.env.MULTIROVER_TARGET_IMAGE || 'ghcr.io/legop3/multiroombarover:latest';
// A registry port is not a tag separator. Strip a digest before splitting the tag.
const configuredName = TARGET_IMAGE.split('@')[0];
const tagSeparator = configuredName.lastIndexOf(':');
const hasConfiguredTag = tagSeparator > configuredName.lastIndexOf('/');
const IMAGE_REPOSITORY = hasConfiguredTag ? configuredName.slice(0, tagSeparator) : configuredName;
const DEFAULT_TAG = TARGET_IMAGE.includes('@') ? null : (hasConfiguredTag ? configuredName.slice(tagSeparator + 1) : 'latest');
const SOCKET_PATH = process.env.MULTIROVER_LIFECYCLE_SOCKET || '/run/multirover/lifecycle.sock';
// Status belongs beside the controller's private socket instead of in the
// application's data volume. The controller runs as root for Docker access;
// keeping it out of /data prevents it from creating directories that the
// non-root application cannot later use for replay and audio runtime work.
const STATUS_PATH = process.env.MULTIROVER_LIFECYCLE_STATUS || '/run/multirover/status.json';
const HEALTH_TIMEOUT_MS = 2 * 60 * 1000;
const HEALTH_POLL_MS = 1000;
let operationRunning = false;
let status = readStatus();
// A repository change in Compose invalidates the previous repository's selection.
if (status.selectedRepository !== IMAGE_REPOSITORY) {
status.selectedRepository = IMAGE_REPOSITORY;
status.selectedTag = DEFAULT_TAG;
}
function selectedImage() {
return status.selectedTag ? `${IMAGE_REPOSITORY}:${status.selectedTag}` : TARGET_IMAGE;
}
function publicStatus() {
return {
...status,
configuredImage: TARGET_IMAGE,
repository: IMAGE_REPOSITORY,
targetImage: selectedImage(),
};
}
function requestedImage(tag) {
if (tag === null) return { imageReference: selectedImage(), tag: status.selectedTag };
if (!/^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$/.test(tag)) {
throw new Error('Enter a valid container tag (1–128 letters, numbers, underscores, periods, or hyphens).');
}
return { imageReference: `${IMAGE_REPOSITORY}:${tag}`, tag };
}
function readStatus() {
try {
const saved = JSON.parse(fs.readFileSync(STATUS_PATH, 'utf8'));
// A controller replacement cannot resume an in-flight Docker operation.
// Preserve its history but make the interrupted result explicit.
if (saved.state === 'running') {
return {
...saved,
state: 'failed',
message: 'The lifecycle controller restarted before the operation completed.',
completedAt: Date.now(),
};
}
return saved;
} catch (error) {
if (error.code !== 'ENOENT') process.stderr.write(`Unable to read lifecycle status: ${error.message}\n`);
return {
state: 'idle',
operation: null,
message: 'No lifecycle operation has run yet.',
startedAt: null,
completedAt: null,
runningImage: null,
availableImage: null,
updateAvailable: null,
rollback: null,
};
}
}
function writeStatus(patch) {
const nextStatus = { ...status, ...patch };
fs.mkdirSync(path.dirname(STATUS_PATH), { recursive: true });
const temporaryPath = `${STATUS_PATH}.tmp`;
fs.writeFileSync(temporaryPath, `${JSON.stringify(nextStatus, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
fs.renameSync(temporaryPath, STATUS_PATH);
status = nextStatus;
return status;
}
function sleep(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
async function inspectTargetContainer() {
return docker.getContainer(TARGET_CONTAINER_NAME).inspect();
}
async function inspectLocalTargetImage(imageReference = selectedImage()) {
try {
return await docker.getImage(imageReference).inspect();
} catch (error) {
if (error.statusCode === 404) return null;
throw error;
}
}
async function pullTargetImage(imageReference = selectedImage()) {
const stream = await new Promise((resolve, reject) => {
docker.pull(imageReference, (error, pullStream) => (error ? reject(error) : resolve(pullStream)));
});
await new Promise((resolve, reject) => {
docker.modem.followProgress(stream, (error) => (error ? reject(error) : resolve()));
});
const image = await inspectLocalTargetImage(imageReference);
if (!image) throw new Error('Docker finished pulling but the target image is unavailable.');
return image;
}
function environmentMap(values = []) {
const result = new Map();
values.forEach((entry) => {
const separator = String(entry).indexOf('=');
const key = separator >= 0 ? entry.slice(0, separator) : entry;
result.set(key, String(entry));
});
return result;
}
function mergeEnvironment({ current = [], previousDefaults = [], nextDefaults = [] }) {
const previous = environmentMap(previousDefaults);
const overrides = current.filter((entry) => previous.get(String(entry).split('=', 1)[0]) !== entry);
const merged = environmentMap(nextDefaults);
overrides.forEach((entry) => merged.set(String(entry).split('=', 1)[0], entry));
return [...merged.values()];
}
async function buildReplacementOptions({ current, imageReference }) {
const [previousImage, nextImage] = await Promise.all([
docker.getImage(current.Image).inspect(),
docker.getImage(imageReference).inspect(),
]);
const imageConfig = nextImage.Config || {};
/*
Image-owned defaults come from the replacement image so updates can change
their command, health check, or runtime environment. Only environment
values that Compose overrode on the running container survive. HostConfig
is copied intact because it contains the named volume, host networking,
device access, capability, security, and restart-policy contract.
*/
return {
name: TARGET_CONTAINER_NAME,
Image: imageReference,
AttachStdin: false,
AttachStdout: false,
AttachStderr: false,
Tty: false,
OpenStdin: false,
StdinOnce: false,
Env: mergeEnvironment({
current: current.Config?.Env,
previousDefaults: previousImage.Config?.Env,
nextDefaults: imageConfig.Env,
}),
Cmd: imageConfig.Cmd,
Entrypoint: imageConfig.Entrypoint,
WorkingDir: imageConfig.WorkingDir,
User: imageConfig.User,
ExposedPorts: imageConfig.ExposedPorts,
Volumes: imageConfig.Volumes,
Healthcheck: imageConfig.Healthcheck,
StopSignal: imageConfig.StopSignal,
StopTimeout: imageConfig.StopTimeout,
Labels: { ...(imageConfig.Labels || {}), ...(current.Config?.Labels || {}) },
HostConfig: current.HostConfig,
};
}
async function removeContainerIfPresent() {
const container = docker.getContainer(TARGET_CONTAINER_NAME);
try {
const current = await container.inspect();
if (current.State?.Running) {
try {
await container.stop({ t: 20 });
} catch (error) {
if (error.statusCode !== 304) throw error;
}
}
await container.remove();
} catch (error) {
if (error.statusCode !== 404) throw error;
}
}
async function replaceContainer({ template, imageReference }) {
const options = await buildReplacementOptions({ current: template, imageReference });
await removeContainerIfPresent();
const replacement = await docker.createContainer(options);
await replacement.start();
return replacement;
}
async function waitForHealthyContainer() {
const deadline = Date.now() + HEALTH_TIMEOUT_MS;
while (Date.now() < deadline) {
const current = await inspectTargetContainer();
const health = current.State?.Health?.Status;
if (current.State?.Running && health === 'healthy') return current;
if (health === 'unhealthy') throw new Error('The replacement container failed its health check.');
await sleep(HEALTH_POLL_MS);
}
throw new Error('The replacement container did not become healthy in time.');
}
async function refreshImageStatus() {
const [container, image] = await Promise.all([
inspectTargetContainer(),
inspectLocalTargetImage(),
]);
return writeStatus({
runningImage: container.Image,
runningImageReference: container.Config?.Image || null,
availableImage: image?.Id || null,
updateAvailable: image ? container.Image !== image.Id : null,
});
}
async function runCheck() {
writeStatus({ message: `Pulling ${selectedImage()}…` });
await pullTargetImage();
const current = await refreshImageStatus();
return current.updateAvailable ? 'An application update is available.' : 'The application is current.';
}
async function runRestart() {
writeStatus({ message: 'Restarting the application container…' });
await docker.getContainer(TARGET_CONTAINER_NAME).restart({ t: 20 });
await waitForHealthyContainer();
await refreshImageStatus();
return 'The application container restarted successfully.';
}
async function runUpdate({ imageReference, tag }) {
writeStatus({ message: `Pulling ${imageReference}…`, rollback: null });
const targetImage = await pullTargetImage(imageReference);
const previousContainer = await inspectTargetContainer();
const previousImage = previousContainer.Image;
let replacementStarted = false;
try {
writeStatus({
message: 'Replacing the application container…',
runningImage: previousImage,
availableImage: targetImage.Id,
updateAvailable: previousImage !== targetImage.Id,
});
replacementStarted = true;
await replaceContainer({ template: previousContainer, imageReference: targetImage.Id });
writeStatus({ message: 'Waiting for the updated application to become healthy…' });
const healthy = await waitForHealthyContainer();
writeStatus({
// Commit the tag only once the replacement passes its health check.
selectedRepository: IMAGE_REPOSITORY,
selectedTag: tag,
runningImage: healthy.Image,
runningImageReference: healthy.Config?.Image || null,
availableImage: targetImage.Id,
updateAvailable: false,
});
return previousImage === targetImage.Id
? 'The current application image was restarted successfully.'
: 'The application was updated successfully.';
} catch (error) {
if (!replacementStarted) throw error;
writeStatus({ message: 'The update failed; restoring the previous application image…' });
try {
await replaceContainer({ template: previousContainer, imageReference: previousImage });
await waitForHealthyContainer();
writeStatus({
rollback: { status: 'succeeded', image: previousImage, completedAt: Date.now() },
runningImage: previousImage,
updateAvailable: true,
});
} catch (rollbackError) {
writeStatus({
rollback: { status: 'failed', image: previousImage, error: rollbackError.message, completedAt: Date.now() },
});
throw new Error(`${error.message} Rollback also failed: ${rollbackError.message}`);
}
throw new Error(`${error.message} The previous image was restored.`);
}
}
function queueOperation(operation, runner) {
if (operationRunning) throw new Error('Another lifecycle operation is already running.');
operationRunning = true;
writeStatus({
state: 'running',
operation,
message: `Starting ${operation}…`,
startedAt: Date.now(),
completedAt: null,
});
// Respond to the application before replacement disconnects its Socket.IO
// clients. The persisted status remains available when the new process asks
// for the result after reconnecting.
setTimeout(async () => {
try {
const message = await runner();
writeStatus({ state: 'succeeded', message, completedAt: Date.now() });
} catch (error) {
writeStatus({ state: 'failed', message: error.message, completedAt: Date.now() });
} finally {
operationRunning = false;
}
}, 500);
return publicStatus();
}
function sendJson(response, statusCode, value) {
response.writeHead(statusCode, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
response.end(JSON.stringify(value));
}
const server = http.createServer((request, response) => {
try {
const url = new URL(request.url, 'http://localhost');
if (request.method === 'GET' && request.url === '/status') {
refreshImageStatus()
.catch(() => status)
.then(() => sendJson(response, 200, { available: true, ...publicStatus() }));
return;
}
if (request.method === 'POST' && request.url === '/check') {
sendJson(response, 202, { available: true, ...queueOperation('check', runCheck) });
return;
}
if (request.method === 'POST' && request.url === '/restart') {
sendJson(response, 202, { available: true, ...queueOperation('restart', runRestart) });
return;
}
if (request.method === 'POST' && url.pathname === '/update') {
const target = requestedImage(url.searchParams.get('tag'));
queueOperation('update', () => runUpdate(target));
sendJson(response, 202, { available: true, ...publicStatus() });
return;
}
sendJson(response, 404, { error: 'Lifecycle operation not found.' });
} catch (error) {
sendJson(response, 409, { error: error.message });
}
});
fs.mkdirSync(path.dirname(SOCKET_PATH), { recursive: true });
fs.rmSync(SOCKET_PATH, { force: true });
server.listen(SOCKET_PATH, () => {
// The application runs as uid 1000 while this Docker-authorized controller
// runs as root. The socket carries only the fixed API above, so permitting
// the application user to connect does not expose arbitrary Docker access.
fs.chmodSync(SOCKET_PATH, 0o666);
process.stdout.write(`Lifecycle controller listening on ${SOCKET_PATH}\n`);
});
function stop() {
server.close(() => {
fs.rmSync(SOCKET_PATH, { force: true });
process.exit(0);
});
}
process.once('SIGINT', stop);
process.once('SIGTERM', stop);