slop

server/src/services/roverLearningService/workers/dataset.py

@@ -47,7 +47,7 @@ def load_events(directory):
     return metadata, sorted(events, key=lambda e: e['ts'])
 
 
-def prepare(directory, schema):
+def prepare(directory, schema, stats):
     metadata, events = load_events(directory)
     requests = {}
     accepted = set()
@@ -72,9 +72,28 @@ def prepare(directory, schema):
     if not sensors:
         raise ValueError('No sensor observations')
     candidate = copy.deepcopy(schema)
-    history, targets = [], []
+    history, targets, wheels = [], [], [(stamp, None) for stamp in boundaries]
+    stats.update({'sensorFrames': len(sensors), 'dispatches': len(dispatches),
+                  'failedCommands': len(failed), 'sensorPresentFraction': np.mean([row[1][32:] for row in sensors], axis=0).tolist()})
     for data in dispatches:
-        encoded = encode_command(data.get('command') or {}, candidate, extend=True)
+        command = data.get('command') or {}
+        request = requests.get(data.get('requestId'))
+        human = (data.get('source') == 'client' and request and data.get('commandId') in accepted
+                 and data.get('commandId') not in failed)
+        stamp = data['ts']
+        if command.get('type') == 'drive':
+            direct = command.get('driveDirect') or {}
+            pair = [direct.get('left'), direct.get('right')]
+            if human and all(isinstance(v, (int, float)) and math.isfinite(v) for v in pair):
+                wheels.append((stamp, np.clip(np.array(pair, dtype=np.float32), -metadata.get('wheelSpeedLimit', 500), metadata.get('wheelSpeedLimit', 500)) / 500))
+            else:
+                wheels.append((stamp, None))
+        elif command.get('type') in ('raw', 'reset', 'reboot', 'update', 'sensorStream'):
+            # Raw OI can change mode or motion; do not assume the last wheel
+            # command survives it. A later explicit human drive resolves intent.
+            wheels.append((stamp, None))
+            boundaries.append(stamp)
+        encoded = encode_command(command, candidate, extend=True)
         if encoded is None:
             continue
         stamp = data['ts']
@@ -93,17 +112,22 @@ def prepare(directory, schema):
     if not targets:
         raise ValueError('No accepted client operating commands')
     schema[:] = candidate
-    return metadata, sensors, sorted(history, key=lambda x: x[0]), sorted(targets, key=lambda x: x[0]), sorted(boundaries)
+    return metadata, sensors, sorted(wheels, key=lambda x: x[0]), sorted(history, key=lambda x: x[0]), sorted(targets, key=lambda x: x[0]), sorted(boundaries)
 
 
-def windows(directory, schema, cancelled=lambda: False):
+def windows(directory, schema, cancelled=lambda: False, stats=None):
     directory = Path(directory)
-    metadata, sensors, history, targets, boundaries = prepare(directory, schema)
+    stats = stats if stats is not None else {}
+    metadata, sensors, wheels, history, targets, boundaries = prepare(directory, schema, stats)
+    wheel_times = [row[0] for row in wheels]
+    stats.update({'usableWindows': 0, 'movingWindows': 0, 'stoppedWindows': 0,
+                  'unknownWheelWindows': 0, 'staleSensorWindows': 0, 'boundaryWindows': 0,
+                  'ambiguousWindows': 0, 'videoFrames': 0, 'maxVideoGapMs': 0})
     sensor_times = [row[0] for row in sensors]
     target_times = [row[0] for row in targets]
     available = np.zeros(SLOTS, dtype=np.float32)
     for _, slot, _ in history:
-        available[slot] = 1
+        available[slot] = float(schema[slot]['template']['type'] != 'drive')
     command_values = np.zeros((SLOTS, VALUES), dtype=np.float32)
     command_times = np.full(SLOTS, -np.inf)
     cursor = 0
@@ -127,6 +151,9 @@ def windows(directory, schema, cancelled=lambda: False):
         stamp = float(frame.pts * frame.time_base) * 1000
         if not math.isfinite(stamp) or stamp < last_pts:
             raise ValueError('Invalid/nonmonotonic video timestamps')
+        stats['videoFrames'] += 1
+        if math.isfinite(last_pts):
+            stats['maxVideoGapMs'] = max(stats['maxVideoGapMs'], stamp - last_pts)
         last_pts = stamp
         if not metadata['startedAt'] - 2000 <= stamp <= metadata['endedAt'] + 2000:
             raise ValueError('Video PTS is not aligned to session Unix time')
@@ -142,6 +169,7 @@ def windows(directory, schema, cancelled=lambda: False):
             cursor += 1
         sensor_index = bisect.bisect_right(sensor_times, stamp) - 1
         if sensor_index < 0 or stamp - sensor_times[sensor_index] > 1000:
+            stats['staleSensorWindows'] += 1
             recent.clear()
             continue
         seen = np.isfinite(command_times).astype(np.float32)
@@ -158,7 +186,14 @@ def windows(directory, schema, cancelled=lambda: False):
         if len(chosen) != 4 or stamp + 100 > metadata['endedAt']:
             continue
         if bisect.bisect_right(boundaries, stamp + 100) > bisect.bisect_left(boundaries, chosen[0][0]):
+            stats['boundaryWindows'] += 1
             continue
+        before = bisect.bisect_left(wheel_times, stamp) - 1
+        after = bisect.bisect_right(wheel_times, stamp + 100) - 1
+        if before < 0 or after < 0 or wheels[before][1] is None or wheels[after][1] is None:
+            stats['unknownWheelWindows'] += 1
+            continue
+        wheel_target, previous_wheels = wheels[after][1], wheels[before][1]
         start = bisect.bisect_left(target_times, stamp)
         end = bisect.bisect_left(target_times, stamp + 100)
         actions = np.zeros(SLOTS, dtype=np.float32)
@@ -168,6 +203,8 @@ def windows(directory, schema, cancelled=lambda: False):
         for _, slot, vector in targets[start:end]:
             template = schema[slot]['template']
             family = template['type']
+            if family == 'drive':
+                continue
             if family == 'peripheral':
                 peripheral = template.get('peripheral') or {}
                 family += ':' + str(peripheral.get('id')) + ':' + str(peripheral.get('control'))
@@ -178,5 +215,8 @@ def windows(directory, schema, cancelled=lambda: False):
             families.add(family)
             actions[slot], values[slot] = 1, vector
         if ambiguous:
+            stats['ambiguousWindows'] += 1
             continue
-        yield np.stack([row[1] for row in chosen]), np.stack([row[2] for row in chosen]), actions, values, available
+        stats['usableWindows'] += 1
+        stats['movingWindows' if np.max(np.abs(wheel_target)) > .02 else 'stoppedWindows'] += 1
+        yield np.stack([row[1] for row in chosen]), np.stack([row[2] for row in chosen]), actions, values, available, wheel_target, previous_wheels

server/src/services/roverLearningService/workers/infer.py

@@ -74,7 +74,9 @@ def main():
             with mutex:
                 while observations and observations[0]['ts'] < stamp:
                     event = observations.popleft()
-                    if event['kind'] == 'sensor':
+                    if event['kind'] == 'settings':
+                        args.threshold = max(.05, min(.99, float(event['threshold'])))
+                    elif event['kind'] == 'sensor':
                         sensors, sensor_time = event['data'], event['ts']
                     elif event['kind'] == 'command':
                         try:
@@ -102,16 +104,19 @@ def main():
                 continue
             before = time.perf_counter()
             with torch.inference_mode():
-                logits, numeric = model(torch.from_numpy(np.stack([row[1] for row in selected])).unsqueeze(0).float() / 255,
+                logits, numeric, wheel_output = model(torch.from_numpy(np.stack([row[1] for row in selected])).unsqueeze(0).float() / 255,
                                         torch.from_numpy(np.stack([row[2] for row in selected])).unsqueeze(0))
                 probabilities = logits[0].sigmoid().tolist()
                 predictions = numeric[0].tolist()
+                wheel_speeds = [round(value * 500) for value in wheel_output[0].tolist()]
             if not all(math.isfinite(value) for value in probabilities):
                 raise ValueError('Nonfinite model output')
             families = {}
             active = set()
             proposals = []
             for slot, descriptor in enumerate(schema):
+                if descriptor['template']['type'] == 'drive':
+                    continue
                 command = json.loads(json.dumps(descriptor['template']))
                 family = command['type']
                 if family == 'peripheral':
@@ -147,8 +152,10 @@ def main():
             for _, _, slot, discrete in families.values():
                 if discrete:
                     latched.add(slot)
+            commands.insert(0, {'type': 'drive', 'driveDirect': dict(zip(('left', 'right'), wheel_speeds))})
             print(json.dumps({'kind': 'prediction', 'frameAt': stamp, 'sensorAt': sensor_time,
                               'latencyMs': (time.perf_counter() - before) * 1000,
+                              'wheelSpeeds': wheel_speeds, 'sensorPresent': sensor_vector(sensors)[32:].tolist(),
                               'commands': commands, 'proposals': proposals, 'threshold': args.threshold}, allow_nan=False), flush=True)
 
 

server/src/services/roverLearningService/workers/policy.py

@@ -12,7 +12,7 @@ import numpy as np
 import torch
 from torch import nn
 
-VERSION = 1
+VERSION = 2
 SLOTS = 64
 VALUES = 8
 FRAMES = 4
@@ -109,6 +109,7 @@ class Policy(nn.Module):
             nn.Linear(128 * 3 * 4, 128), nn.ReLU(),
         )
         self.history = nn.GRU(128 + STATE_SIZE, 128, batch_first=True)
+        self.wheels = nn.Linear(128, 2)
         self.actions = nn.Linear(128, SLOTS)
         self.values = nn.Linear(128, SLOTS * VALUES)
 
@@ -117,15 +118,17 @@ class Policy(nn.Module):
         encoded = self.vision(images.reshape(-1, 3, HEIGHT, WIDTH)).reshape(batch, frames, -1)
         history, _ = self.history(torch.cat((encoded, state), dim=-1))
         last = history[:, -1]
-        return self.actions(last), self.values(last).reshape(batch, SLOTS, VALUES).tanh()
+        return self.actions(last), self.values(last).reshape(batch, SLOTS, VALUES).tanh(), self.wheels(last).tanh()
 
 
 def specification(schema):
-    return {'version': VERSION, 'architecture': 'spatial-cnn-gru-v1',
+    return {'version': VERSION, 'architecture': 'spatial-cnn-gru-wheels-v2',
             'frames': FRAMES, 'height': HEIGHT, 'width': WIDTH,
             'historyStrideMs': 300, 'decisionIntervalMs': 100,
             'stateSize': STATE_SIZE, 'sensorFields': SENSORS,
             'normalization': '2/pi * atan(value/scale); missing sensors have mask=0',
             'maxCommandShapes': SLOTS, 'numericFieldsPerShape': VALUES,
-            'commands': schema, 'actionSemantics': 'Events in the next 100 ms; zero means no new command',
+            'commands': schema, 'actionSemantics': 'Continuous wheels at +100ms; accessories are events in the next 100ms',
+            'wheelScaleMmPerSecond': 500, 'wheelOrder': ['left', 'right'],
+            'wheelCommandExpiryMs': None, 'unknownWheelIntent': 'Excluded until next accepted human drive',
             'timing': 'Server-receipt alignment; browser display latency unmeasured'}

server/src/services/roverLearningService/workers/train.py

@@ -11,6 +11,7 @@ import json
 import os
 from pathlib import Path
 import random
+import resource
 import signal
 import time
 import uuid
@@ -70,7 +71,7 @@ def main():
     model = Policy().cpu()
     optimizer = torch.optim.AdamW(model.parameters(), lr=0.0001)
     # A fresh training lineage prevents previously trained footage leaking into validation.
-    resume_path = root / 'training' / 'resume-v2.pt'
+    resume_path = root / 'training' / 'resume-v3.pt'
     schema, steps, examples, last_published = [], 0, 0, 0
     trained_rovers = set()
     control_profiles = []
@@ -89,6 +90,9 @@ def main():
         control_profiles = saved.get('controlProfiles', [])
         distinct_windows = saved.get('distinctWindows', {})
         published_windows = saved.get('publishedWindows', 0)
+    cpu_started = time.process_time()
+    decode_started = time.monotonic()
+    dataset_stats = []
     initial_steps = steps
     pool, validation, accepted, rejected = [], [], [], {}
     observed_windows = {}
@@ -102,18 +106,34 @@ def main():
             break
         candidate_schema = copy.deepcopy(schema)
         samples, seen = [], 0
+        stats = {}
+        buckets = {key: [] for key in ('stopped', 'forward', 'reverse', 'turn')}
+        counts = dict.fromkeys(buckets, 0)
+        held_out = is_held_out(session_id)
         try:
             directory = root / 'recordings' / session_id
-            for sample in windows(directory, candidate_schema, cancelled):
+            for sample in windows(directory, candidate_schema, cancelled, stats):
                 seen += 1
-                # Uniform reservoir across the whole recording, not just the
-                # opening seconds. Frames remain uint8 until a batch is used.
-                if len(samples) < per_session:
-                    samples.append(sample)
+                # Keep evaluation naturally distributed. Training reserves space
+                # for ordinary turns/reversing so idle or straight motion cannot
+                # consume the entire sample budget. No scripted driving required.
+                if held_out:
+                    reservoir, count, capacity = samples, seen, per_session
                 else:
-                    index = random.randrange(seen)
-                    if index < per_session:
-                        samples[index] = sample
+                    left, right = sample[5]
+                    category = ('stopped' if max(abs(left), abs(right)) <= .02 else
+                                'turn' if abs(left - right) > .1 else
+                                'reverse' if left + right < 0 else 'forward')
+                    counts[category] += 1
+                    reservoir, count, capacity = buckets[category], counts[category], max(1, per_session // 4)
+                if len(reservoir) < capacity:
+                    reservoir.append(sample)
+                else:
+                    index = random.randrange(count)
+                    if index < capacity:
+                        reservoir[index] = sample
+            if not held_out:
+                samples = [sample for bucket in buckets.values() for sample in bucket]
             if cancelled():
                 break  # Do not call a partially decoded recording validated.
             if not samples:
@@ -123,6 +143,8 @@ def main():
             held_out = is_held_out(session_id)
             if held_out:
                 if candidate_schema != schema:
+                    dataset_stats.append({**stats, 'partition': 'heldOut', 'sampledWindows': len(samples),
+                                          'skipReason': 'Vocabulary absent from training data'})
                     continue  # Retry after training data introduces these command shapes.
                 validation.extend(samples)
             else:
@@ -136,15 +158,20 @@ def main():
             profile = metadata.get('controlProfile')
             if not held_out and profile is not None and profile not in control_profiles:
                 control_profiles.append(profile)
-            emit('dataset', sessionId=session_id, windows=seen, sampled=len(samples))
+            stats.update({'partition': 'heldOut' if held_out else 'training', 'sampledWindows': len(samples)})
+            dataset_stats.append(stats)
+            emit('dataset', sessionId=session_id, windows=seen, sampled=len(samples), diagnostics=stats)
         except (ValueError, KeyError, TypeError, OSError, EOFError, IndexError, OverflowError, av.error.FFmpegError) as error:
             rejected[session_id] = str(error)[:500]
             emit('skipped', sessionId=session_id, reason=rejected[session_id])
-    if len(pool) < config['minimumSamples'] or not any(sample[2].any() for sample in pool) or cancelled():
+    if len(pool) < config['minimumSamples'] or not any(np.max(np.abs(sample[5])) > .02 for sample in pool) or cancelled():
         emit('complete', trained=False, accepted=[], rejected=rejected, samples=len(pool),
+             diagnostics={'dataset': dataset_stats, 'decodeSeconds': time.monotonic() - decode_started},
              reason='Stopped/time limit' if cancelled() else 'Waiting for enough training-partition samples with human actions')
         return
 
+    decode_seconds = time.monotonic() - decode_started
+    train_started = time.monotonic()
     numeric_mask = torch.zeros(SLOTS, VALUES)
     for slot, descriptor in enumerate(schema):
         numeric_mask[slot, :len(descriptor['fields'])] = 1
@@ -161,16 +188,17 @@ def main():
             if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
                 break
             batch = pool[offset:offset + config['batchSize']]
-            images, states, targets, values, available = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
+            images, states, targets, values, available, wheel_targets, previous_wheels = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
             images = images.float().div_(255)
             optimizer.zero_grad(set_to_none=True)
-            action_logits, predictions = model(images, states)
+            action_logits, predictions, wheel_predictions = model(images, states)
             event_loss = functional.binary_cross_entropy_with_logits(action_logits, targets,
                                                                        pos_weight=positive_weight, reduction='none')
             event_loss = (event_loss * available).sum() / available.sum().clamp_min(1)
             mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
             value_loss = ((predictions - values).square() * mask).sum() / mask.sum().clamp_min(1)
-            loss = event_loss + value_loss
+            wheel_loss = functional.smooth_l1_loss(wheel_predictions, wheel_targets)
+            loss = wheel_loss + event_loss + value_loss
             if not torch.isfinite(loss):
                 raise ValueError('Nonfinite training loss; checkpoint not saved')
             loss.backward()
@@ -180,13 +208,16 @@ def main():
             examples += len(batch)
             losses.append(loss.item())
             if steps % 10 == 0:
-                emit('progress', steps=steps, examples=examples, loss=loss.item())
+                emit('progress', steps=steps, examples=examples, loss=loss.item(),
+                     losses={'wheels': wheel_loss.item(), 'events': event_loss.item(), 'accessoryValues': value_loss.item()})
         if cancelled() or steps - initial_steps >= config['maxStepsPerJob']:
             break
     if steps == initial_steps:
         emit('complete', trained=False, accepted=[], rejected=rejected, reason='No training steps completed')
         return
 
+    training_seconds = time.monotonic() - train_started
+    evaluation_started = time.monotonic()
     distinct_windows.update(observed_windows)
     unique_count = sum(distinct_windows.values())
     evaluation = {'status': 'waiting for held-out sessions', 'windows': len(validation)}
@@ -195,13 +226,15 @@ def main():
         true_positive = predicted_positive = actual_positive = 0
         value_error = value_count = 0
         evaluated = 0
+        wheel_error = stopped_error = persistence_error = 0
+        changed_error = changed_persistence = changed_count = moving_count = 0
         with torch.inference_mode():
             for offset in range(0, len(validation), config['batchSize']):
                 if cancelled():
                     break
                 batch = validation[offset:offset + config['batchSize']]
-                images, states, targets, values, available = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
-                logits, predictions = model(images.float() / 255, states)
+                images, states, targets, values, available, wheel_targets, previous_wheels = [torch.from_numpy(np.stack(items)) for items in zip(*batch)]
+                logits, predictions, wheel_predictions = model(images.float() / 255, states)
                 predicted = (logits.sigmoid() >= job['actionThreshold']) & available.bool()
                 actual = targets.bool() & available.bool()
                 true_positive += (predicted & actual).sum().item()
@@ -210,16 +243,42 @@ def main():
                 mask = numeric_mask.unsqueeze(0) * targets.unsqueeze(-1)
                 value_error += ((predictions - values).square() * mask).sum().item()
                 value_count += mask.sum().item()
+                errors = (wheel_predictions - wheel_targets).abs().mean(dim=1) * 500
+                persistence = (previous_wheels - wheel_targets).abs().mean(dim=1) * 500
+                changed = (previous_wheels - wheel_targets).abs().amax(dim=1) > .02
+                changed_error += errors[changed].sum().item()
+                changed_persistence += persistence[changed].sum().item()
+                changed_count += changed.sum().item()
+                moving_count += (wheel_targets.abs().amax(dim=1) > .02).sum().item()
+                wheel_error += errors.sum().item()
+                stopped_error += wheel_targets.abs().mean(dim=1).sum().item() * 500
+                persistence_error += persistence.sum().item()
                 evaluated += len(batch)
         precision = true_positive / max(1, predicted_positive)
         recall = true_positive / max(1, actual_positive)
         mse = value_error / max(1, value_count)
-        # A basic imitation gate, not an autonomous-driving certification.
-        passed = (evaluated == len(validation) and evaluated >= 128 and actual_positive >= 32
-                  and precision >= .5 and recall >= .5 and value_count > 0 and mse <= .1)
+        wheel_mae = wheel_error / max(1, evaluated)
+        stopped_mae = stopped_error / max(1, evaluated)
+        persistence_mae = persistence_error / max(1, evaluated)
+        changed_mae = changed_error / max(1, changed_count)
+        changed_baseline = changed_persistence / max(1, changed_count)
+        # Check control changes separately: copying the previous speed can look
+        # excellent on long straight runs without learning to steer or stop.
+        wheels_pass = (evaluated == len(validation) and evaluated >= 128 and moving_count >= 32
+                       and changed_count >= 32 and wheel_mae < stopped_mae
+                       and wheel_mae <= persistence_mae and changed_mae < changed_baseline)
+        accessories_pass = (predicted_positive == 0 if actual_positive == 0 else
+                            actual_positive >= 32 and precision >= .5 and recall >= .5)
+        passed = wheels_pass and accessories_pass and (value_count == 0 or mse <= .1)
         evaluation = {'status': 'passed imitation gate' if passed else 'imitation gate not met',
                       'windows': evaluated, 'precision': precision, 'recall': recall,
-                      'numericMse': mse, 'positiveActions': actual_positive}
+                      'numericMse': mse, 'positiveActions': actual_positive,
+                      'wheelMaeMmPerSecond': wheel_mae, 'stoppedBaselineMaeMmPerSecond': stopped_mae,
+                      'previousSpeedBaselineMaeMmPerSecond': persistence_mae,
+                      'changedWheelMaeMmPerSecond': changed_mae, 'changedBaselineMaeMmPerSecond': changed_baseline,
+                      'changedWindows': changed_count, 'movingWindows': moving_count,
+                      'wheelsPassed': wheels_pass, 'accessoriesPassed': accessories_pass}
+    evaluation_seconds = time.monotonic() - evaluation_started
     publication = {'distinctMinutes': unique_count / 600,
                    'newMinutes': (unique_count - published_windows) / 600,
                    'minimumMinutes': config['minimumDrivingMinutes'],
@@ -236,6 +295,7 @@ def main():
     if cancelled():
         reasons.append('Job time budget reached')
     publication['reason'] = '; '.join(reasons) or 'Published'
+    save_started = time.monotonic()
     published = None
     if not reasons:
         adjectives = ['amber', 'curious', 'gentle', 'quiet', 'silver', 'bright', 'merry', 'sleepy']
@@ -270,9 +330,15 @@ def main():
         'controlProfiles': control_profiles, 'distinctWindows': distinct_windows,
         'publishedWindows': published_windows,
     }, resume_path)
+    diagnostics = {'dataset': dataset_stats, 'decodeSeconds': decode_seconds,
+                   'trainingSeconds': training_seconds, 'evaluationSeconds': evaluation_seconds, 'checkpointSeconds': time.monotonic() - save_started,
+                   'cpuSeconds': time.process_time() - cpu_started,
+                   'peakRssMiB': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
+                   'dependencies': {'torch': str(torch.__version__), 'numpy': str(np.__version__), 'av': str(av.__version__)},
+                   'policy': specification(schema), 'browserVideoLatencyMs': None}
     emit('complete', trained=True, accepted=accepted, rejected=rejected,
          steps=steps, examples=examples, loss=sum(losses) / len(losses),
-         published=published, publication=publication, evaluation=evaluation, elapsedSeconds=time.monotonic() - started)
+         published=published, publication=publication, evaluation=evaluation, diagnostics=diagnostics, elapsedSeconds=time.monotonic() - started)
 
 
 if __name__ == '__main__':