Files
MultiRoombaRover/server/src/services/roverLearningService/workers/train.py
T
legop3 008c90edf1
Container image / image (push) Failing after 59s
slop
2026-09-28 14:46:40 -04:00

353 lines
19 KiB
Python

#!/usr/bin/env python3
"""One finite CPU training job; Node owns scheduling, leases, and shutdown.
Usage: python train.py --job /path/to/job.json
Dependencies are packaged at image build time, never installed at runtime.
"""
import argparse
import copy
import hashlib
import json
import os
from pathlib import Path
import random
import resource
import signal
import time
import uuid
def emit(kind, **data):
print(json.dumps({'kind': kind, **data}, allow_nan=False), flush=True)
def atomic_torch_save(torch, value, path):
temporary = path.with_suffix('.tmp')
with temporary.open('wb') as stream:
torch.save(value, stream)
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--job')
parser.add_argument('--check-dependencies', action='store_true', help='Describe the installed production policy and exit without creating storage')
args = parser.parse_args()
# --help works even before Python dependencies are installed.
import numpy as np
import av
import torch
from torch.nn import functional as functional
from dataset import windows
from policy import Policy, VERSION, SLOTS, VALUES, specification
if args.check_dependencies:
emit('dependencies', torch=str(torch.__version__), numpy=str(np.__version__), av=str(av.__version__),
parameters=sum(p.numel() for p in Policy().parameters()), specification=specification([]))
return
if not args.job:
parser.error('--job is required unless --check-dependencies is used')
job = json.loads(Path(args.job).read_text())
root = Path(job['root'])
config = job['config']
torch.set_num_threads(config['threads'])
torch.set_num_interop_threads(1)
# Training yields CPU priority to the Node/media processes. Thread count is
# an additional bound, not a claim to reserve specific physical cores.
os.nice(10)
stopped = False
def stop(_signal, _frame):
nonlocal stopped
stopped = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
started = time.monotonic()
cancelled = lambda: stopped or time.monotonic() - started >= config['maxJobSeconds']
model = Policy().cpu()
optimizer = torch.optim.AdamW(model.parameters(), lr=0.0001)
# A fresh training lineage prevents previously trained footage leaking into validation.
resume_path = root / 'training' / 'resume-v3.pt'
schema, steps, examples, last_published = [], 0, 0, 0
trained_rovers = set()
control_profiles = []
distinct_windows = {}
published_windows = 0
if resume_path.exists():
saved = torch.load(resume_path, map_location='cpu', weights_only=True)
if saved['version'] != VERSION:
raise ValueError('Resume schema incompatible; refusing to reinterpret existing weights')
model.load_state_dict(saved['weights'])
optimizer.load_state_dict(saved['optimizer'])
schema = saved['schema']
steps, examples = saved['steps'], saved['examples']
last_published = saved['lastPublished']
trained_rovers = set(saved['trainedRovers'])
control_profiles = saved.get('controlProfiles', [])
distinct_windows = saved.get('distinctWindows', {})
published_windows = saved.get('publishedWindows', 0)
cpu_started = time.process_time()
decode_started = time.monotonic()
dataset_stats = []
initial_steps = steps
pool, validation, accepted, rejected = [], [], [], {}
observed_windows = {}
sessions = list(job['sessions'])
random.shuffle(sessions)
is_held_out = lambda session_id: int(hashlib.sha256(session_id.encode()).hexdigest()[:8], 16) % 5 == 0
sessions.sort(key=is_held_out)
per_session = max(1, config['maxSamples'] // max(1, len(sessions)))
for session_id in sessions:
if cancelled():
break
candidate_schema = copy.deepcopy(schema)
samples, seen = [], 0
stats = {}
buckets = {key: [] for key in ('stopped', 'forward', 'reverse', 'turn')}
counts = dict.fromkeys(buckets, 0)
held_out = is_held_out(session_id)
try:
directory = root / 'recordings' / session_id
for sample in windows(directory, candidate_schema, cancelled, stats):
seen += 1
# Keep evaluation naturally distributed. Training reserves space
# for ordinary turns/reversing so idle or straight motion cannot
# consume the entire sample budget. No scripted driving required.
if held_out:
reservoir, count, capacity = samples, seen, per_session
else:
left, right = sample[5]
category = ('stopped' if max(abs(left), abs(right)) <= .02 else
'turn' if abs(left - right) > .1 else
'reverse' if left + right < 0 else 'forward')
counts[category] += 1
reservoir, count, capacity = buckets[category], counts[category], max(1, per_session // 4)
if len(reservoir) < capacity:
reservoir.append(sample)
else:
index = random.randrange(count)
if index < capacity:
reservoir[index] = sample
if not held_out:
samples = [sample for bucket in buckets.values() for sample in bucket]
if cancelled():
break # Do not call a partially decoded recording validated.
if not samples:
raise ValueError('No complete, aligned observation/action windows')
# Entire sessions stay in one partition across all jobs. Neighboring
# video windows must never straddle training and validation.
held_out = is_held_out(session_id)
if held_out:
if candidate_schema != schema:
dataset_stats.append({**stats, 'partition': 'heldOut', 'sampledWindows': len(samples),
'skipReason': 'Vocabulary absent from training data'})
continue # Retry after training data introduces these command shapes.
validation.extend(samples)
else:
schema = candidate_schema
pool.extend(samples)
observed_windows[session_id] = seen
accepted.append(session_id)
metadata = json.loads((directory / 'session.json').read_text())
if not held_out:
trained_rovers.add(str(metadata['roverId']))
profile = metadata.get('controlProfile')
if not held_out and profile is not None and profile not in control_profiles:
control_profiles.append(profile)
stats.update({'partition': 'heldOut' if held_out else 'training', 'sampledWindows': len(samples)})
dataset_stats.append(stats)
emit('dataset', sessionId=session_id, windows=seen, sampled=len(samples), diagnostics=stats)
except (ValueError, KeyError, TypeError, OSError, EOFError, IndexError, OverflowError, av.error.FFmpegError) as error:
rejected[session_id] = str(error)[:500]
emit('skipped', sessionId=session_id, reason=rejected[session_id])
if len(pool) < config['minimumSamples'] or not any(np.max(np.abs(sample[5])) > .02 for sample in pool) or cancelled():
emit('complete', trained=False, accepted=[], rejected=rejected, samples=len(pool),
diagnostics={'dataset': dataset_stats, 'decodeSeconds': time.monotonic() - decode_started},
reason='Stopped/time limit' if cancelled() else 'Waiting for enough training-partition samples with human actions')
return
decode_seconds = time.monotonic() - decode_started
train_started = time.monotonic()
numeric_mask = torch.zeros(SLOTS, VALUES)
for slot, descriptor in enumerate(schema):
numeric_mask[slot, :len(descriptor['fields'])] = 1
# Balance sparse events without assuming that every time step should fire
# every control. This is training loss, not a driving-quality evaluation.
positives = np.sum([sample[2] for sample in pool], axis=0)
opportunities = np.sum([sample[4] for sample in pool], axis=0)
positive_weight = torch.from_numpy(np.clip((opportunities - positives) / np.maximum(positives, 1), 1, 20)).float()
losses = []
model.train()
for _ in range(config['passesPerJob']):
random.shuffle(pool)
for offset in range(0, len(pool), config['batchSize']):
if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
break
batch = pool[offset:offset + config['batchSize']]
images, states, targets, values, available, wheel_targets, previous_wheels = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
images = images.float().div_(255)
optimizer.zero_grad(set_to_none=True)
action_logits, predictions, wheel_predictions = model(images, states)
event_loss = functional.binary_cross_entropy_with_logits(action_logits, targets,
pos_weight=positive_weight, reduction='none')
event_loss = (event_loss * available).sum() / available.sum().clamp_min(1)
mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
value_loss = ((predictions - values).square() * mask).sum() / mask.sum().clamp_min(1)
wheel_loss = functional.smooth_l1_loss(wheel_predictions, wheel_targets)
loss = wheel_loss + event_loss + value_loss
if not torch.isfinite(loss):
raise ValueError('Nonfinite training loss; checkpoint not saved')
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
optimizer.step()
steps += 1
examples += len(batch)
losses.append(loss.item())
if steps % 10 == 0:
emit('progress', steps=steps, examples=examples, loss=loss.item(),
losses={'wheels': wheel_loss.item(), 'events': event_loss.item(), 'accessoryValues': value_loss.item()})
if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
break
if steps == initial_steps:
emit('complete', trained=False, accepted=[], rejected=rejected, reason='No training steps completed')
return
training_seconds = time.monotonic() - train_started
evaluation_started = time.monotonic()
distinct_windows.update(observed_windows)
unique_count = sum(distinct_windows.values())
evaluation = {'status': 'waiting for held-out sessions', 'windows': len(validation)}
if validation and not cancelled():
model.eval()
true_positive = predicted_positive = actual_positive = 0
value_error = value_count = 0
evaluated = 0
wheel_error = stopped_error = persistence_error = 0
changed_error = changed_persistence = changed_count = moving_count = 0
with torch.inference_mode():
for offset in range(0, len(validation), config['batchSize']):
if cancelled():
break
batch = validation[offset:offset + config['batchSize']]
images, states, targets, values, available, wheel_targets, previous_wheels = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
logits, predictions, wheel_predictions = model(images.float() / 255, states)
predicted = (logits.sigmoid() >= job['actionThreshold']) & available.bool()
actual = targets.bool() & available.bool()
true_positive += (predicted & actual).sum().item()
predicted_positive += predicted.sum().item()
actual_positive += actual.sum().item()
mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
value_error += ((predictions - values).square() * mask).sum().item()
value_count += mask.sum().item()
errors = (wheel_predictions - wheel_targets).abs().mean(dim=1) * 500
persistence = (previous_wheels - wheel_targets).abs().mean(dim=1) * 500
changed = (previous_wheels - wheel_targets).abs().amax(dim=1) > .02
changed_error += errors[changed].sum().item()
changed_persistence += persistence[changed].sum().item()
changed_count += changed.sum().item()
moving_count += (wheel_targets.abs().amax(dim=1) > .02).sum().item()
wheel_error += errors.sum().item()
stopped_error += wheel_targets.abs().mean(dim=1).sum().item() * 500
persistence_error += persistence.sum().item()
evaluated += len(batch)
precision = true_positive / max(1, predicted_positive)
recall = true_positive / max(1, actual_positive)
mse = value_error / max(1, value_count)
wheel_mae = wheel_error / max(1, evaluated)
stopped_mae = stopped_error / max(1, evaluated)
persistence_mae = persistence_error / max(1, evaluated)
changed_mae = changed_error / max(1, changed_count)
changed_baseline = changed_persistence / max(1, changed_count)
# Check control changes separately: copying the previous speed can look
# excellent on long straight runs without learning to steer or stop.
wheels_pass = (evaluated == len(validation) and evaluated >= 128 and moving_count >= 32
and changed_count >= 32 and wheel_mae < stopped_mae
and wheel_mae <= persistence_mae and changed_mae < changed_baseline)
accessories_pass = (predicted_positive == 0 if actual_positive == 0 else
actual_positive >= 32 and precision >= .5 and recall >= .5)
passed = wheels_pass and accessories_pass and (value_count == 0 or mse <= .1)
evaluation = {'status': 'passed imitation gate' if passed else 'imitation gate not met',
'windows': evaluated, 'precision': precision, 'recall': recall,
'numericMse': mse, 'positiveActions': actual_positive,
'wheelMaeMmPerSecond': wheel_mae, 'stoppedBaselineMaeMmPerSecond': stopped_mae,
'previousSpeedBaselineMaeMmPerSecond': persistence_mae,
'changedWheelMaeMmPerSecond': changed_mae, 'changedBaselineMaeMmPerSecond': changed_baseline,
'changedWindows': changed_count, 'movingWindows': moving_count,
'wheelsPassed': wheels_pass, 'accessoriesPassed': accessories_pass}
evaluation_seconds = time.monotonic() - evaluation_started
publication = {'distinctMinutes': unique_count / 600,
'newMinutes': (unique_count - published_windows) / 600,
'minimumMinutes': config['minimumDrivingMinutes'],
'requiredNewMinutes': config['newDrivingMinutesPerModel']}
reasons = []
if publication['distinctMinutes'] < config['minimumDrivingMinutes']:
reasons.append('Collecting distinct driving footage')
if publication['newMinutes'] < config['newDrivingMinutesPerModel']:
reasons.append('Waiting for new driving footage since last publication')
if steps - last_published < config['checkpointEverySteps']:
reasons.append('Training steps below publication interval')
if evaluation['status'] != 'passed imitation gate':
reasons.append(evaluation['status'])
if cancelled():
reasons.append('Job time budget reached')
publication['reason'] = '; '.join(reasons) or 'Latest updated'
save_started = time.monotonic()
published = None
# Every completed job exports an immutable candidate; Node owns promotion and retention.
adjectives = ['amber', 'curious', 'gentle', 'quiet', 'silver', 'bright', 'merry', 'sleepy']
animals = ['otter', 'finch', 'badger', 'robin', 'fox', 'heron', 'marten', 'wren']
model_id = str(uuid.uuid4())
name = f'{random.choice(adjectives)}-{random.choice(animals)}-{model_id[:8]}'
checkpoint_root = root / 'training' / 'checkpoints'
checkpoint_root.mkdir(exist_ok=True)
staging = checkpoint_root / f'.{model_id}.pending'
staging.mkdir()
weights_path = staging / 'weights.pt'
atomic_torch_save(torch, model.state_dict(), weights_path)
published = {
'id': model_id, 'name': name, 'createdAt': int(time.time() * 1000),
'trainingSteps': steps, 'examplesProcessed': examples,
'parameters': sum(p.numel() for p in model.parameters()),
'weightBytes': weights_path.stat().st_size,
'sha256': hashlib.sha256(weights_path.read_bytes()).hexdigest(),
'specification': specification(schema), 'trainedRovers': sorted(trained_rovers),
'controlProfiles': control_profiles,
'sourceSessions': accepted, 'trainingLoss': sum(losses) / len(losses),
'evaluation': evaluation['status'], 'validation': evaluation, 'distinctMinutes': publication['distinctMinutes'], 'dtype': 'float32', 'torchVersion': str(torch.__version__),
}
(staging / 'model.json').write_text(json.dumps(published, indent=2, allow_nan=False))
# A catalog reader sees both weights and metadata, or neither. Existing
# versions are never overwritten or deleted by the trainer.
staging.rename(checkpoint_root / model_id)
if not reasons:
last_published = steps
published_windows = unique_count
atomic_torch_save(torch, {
'version': VERSION, 'weights': model.state_dict(), 'optimizer': optimizer.state_dict(),
'schema': schema, 'steps': steps, 'examples': examples,
'lastPublished': last_published, 'trainedRovers': sorted(trained_rovers),
'controlProfiles': control_profiles, 'distinctWindows': distinct_windows,
'publishedWindows': published_windows,
}, resume_path)
diagnostics = {'dataset': dataset_stats, 'decodeSeconds': decode_seconds,
'trainingSeconds': training_seconds, 'evaluationSeconds': evaluation_seconds, 'checkpointSeconds': time.monotonic() - save_started,
'cpuSeconds': time.process_time() - cpu_started,
'peakRssMiB': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
'dependencies': {'torch': str(torch.__version__), 'numpy': str(np.__version__), 'av': str(av.__version__)},
'policy': specification(schema), 'browserVideoLatencyMs': None}
emit('complete', trained=True, accepted=accepted, rejected=rejected,
steps=steps, examples=examples, loss=sum(losses) / len(losses),
published=published, promote=not reasons, publication=publication, evaluation=evaluation, diagnostics=diagnostics, elapsedSeconds=time.monotonic() - started)
if __name__ == '__main__':
try:
main()
except Exception as error:
emit('error', message=str(error))
raise SystemExit(1)