From 4284014578c0babfdc04ea132725718172a3e689 Mon Sep 17 00:00:00 2001 From: legop3 Date: Thu, 26 Feb 2026 18:06:50 -0500 Subject: [PATCH] soup --- server/prompts/commentary_system.txt | 13 +- server/src/services/chatService.js | 42 ++++ server/src/services/llmCommentaryService.js | 210 +++++++++++++++++++- 3 files changed, 254 insertions(+), 11 deletions(-) diff --git a/server/prompts/commentary_system.txt b/server/prompts/commentary_system.txt index 48d79b31..cd9d1aff 100644 --- a/server/prompts/commentary_system.txt +++ b/server/prompts/commentary_system.txt @@ -21,10 +21,20 @@ Direct-address override (strict): Conversation format you will receive: - System message (this prompt). - User RUN META message (plain text bullets). -- Ordered timeline of user CHAT messages and assistant bot messages. +- Ordered timeline of user CHAT messages, user EVENT messages, and assistant bot messages. - Optional user SNAPSHOT message(s) inserted in the timeline. - Final user SNAPSHOT FINAL message with full current rover state. +CHAT rover context format: +- `rover_now` uses compact categorical context: +- status, battery_low, docked, charging, wheels_off_ground, contact, hazard, mobility, activity_band, activity_trend +- `activity_score` may also be present for internal significance checks. +- Treat these as qualitative state tags, not metrics. + +EVENT messages: +- EVENT messages represent major rover state changes (for example dock/undock, battery_low changes, wheels_off_ground changes). +- Treat EVENT messages as high-priority narrative anchors. + Decision policy: - SKIP is the default. - Only speak for clearly notable, new moments from either chat OR rover activity. @@ -84,6 +94,7 @@ Banned filler patterns: Numeric policy: - Numeric fields are internal context only. - Never directly quote counters, percentages, or timers. +- Never directly quote activity_score. - Use numbers only to judge significance. Style: diff --git a/server/src/services/chatService.js b/server/src/services/chatService.js index 1d910e58..b0bf6e71 100644 --- a/server/src/services/chatService.js +++ b/server/src/services/chatService.js @@ -141,6 +141,42 @@ function buildRoverCtxSnapshot(roverId) { const latestBumps = (sensors?.bumpsAndWheelDrops?.bumpLeft ? 0.5 : 0) + (sensors?.bumpsAndWheelDrops?.bumpRight ? 0.5 : 0); + const light = sensors?.lightBumper || {}; + const contactState = + latestBumps >= 0.5 + ? 'bumps_recent' + : sensors?.wall || light.left || light.frontLeft || light.centerLeft || light.centerRight || light.frontRight || light.right + ? 'wall_brush' + : 'clear'; + const hazardState = + sensors?.virtualWall + ? 'virtual_wall_seen' + : sensors?.cliffLeft || sensors?.cliffFrontLeft || sensors?.cliffFrontRight || sensors?.cliffRight + ? 'cliff_alert' + : 'normal'; + const mobilityState = wheelsOffGround ? 'wheels_off_ground' : 'normal'; + const baseScore = Math.min(100, Math.round(Math.min(45, latestDistanceM * 25) + Math.min(30, latestTurnDeg / 12) + Math.min(25, latestBumps * 12))); + const activityScore = Math.max( + 0, + Math.min( + 100, + baseScore + + (contactState === 'wall_brush' ? 6 : 0) + + (contactState === 'bumps_recent' ? 12 : 0) + + (hazardState !== 'normal' ? 8 : 0) + + (wheelsOffGround ? -20 : 0), + ), + ); + const activityBand = + activityScore >= 75 + ? 'intense' + : activityScore >= 50 + ? 'high' + : activityScore >= 25 + ? 'medium' + : activityScore >= 8 + ? 'low' + : 'idle'; const moving = latestDistanceM > 0.05 || latestTurnDeg > 10; let statusTag = 'idle'; if (charging) { @@ -159,6 +195,12 @@ function buildRoverCtxSnapshot(roverId) { docked, charging, wheels_off_ground: wheelsOffGround, + contact_state: contactState, + hazard_state: hazardState, + mobility_state: mobilityState, + activity_score: activityScore, + activity_band: activityBand, + activity_trend: 'steady', activity_30s: { distance_m: latestDistanceM, turn_deg: latestTurnDeg, diff --git a/server/src/services/llmCommentaryService.js b/server/src/services/llmCommentaryService.js index 1904c054..16f55d4e 100644 --- a/server/src/services/llmCommentaryService.js +++ b/server/src/services/llmCommentaryService.js @@ -18,11 +18,13 @@ const MAX_ROVERS = 6; const MAX_CHAT_MESSAGES = 4; const MAX_BOT_MESSAGES = 1; const SKIP_TOKEN = 'SKIP'; -const ACTIVITY_WINDOW_MS = 30000; +const ACTIVITY_WINDOW_MS = 60000; const ACTIVITY_BUCKET_MS = 1000; +const ACTIVITY_SCORE_WINDOW_MS = 30000; const SELF_TALK_WINDOW_MS = 30 * 60 * 1000; const MAX_CONTEXT_EVENTS = 10; const MAX_RUN_HISTORY = 30; +const MAX_ROVER_EVENTS = 400; const config = loadConfig(); const commentaryConfig = config.llmCommentary || {}; @@ -44,6 +46,8 @@ let clearCount = 0; let runHistory = []; let currentRun = null; const roverActivity = new Map(); // roverId -> { buckets: Map(bucketTs -> { distanceMm, turnDeg, bumps }), bumpLeftActive, bumpRightActive } +const roverMajorEvents = []; // [{ ts, type: 'event', event_type, rover_id, driver_nickname, summary }] +const lastSensorFlagsByRover = new Map(); // roverId -> { docked, charging, battery_low, wheels_off_ground } const lastRoverStateById = new Map(); // roverId -> compact rover state used as prev_state function normalizeFrequencyMs(value) { @@ -243,6 +247,13 @@ function updateStatus(patch = {}) { emitStatusToAdmins(); } +function pushRoverMajorEvent(event) { + roverMajorEvents.push(event); + if (roverMajorEvents.length > MAX_ROVER_EVENTS) { + roverMajorEvents.shift(); + } +} + function pruneActivityBuckets(state, nowMs) { if (!state?.buckets) return; const minTs = nowMs - ACTIVITY_WINDOW_MS; @@ -264,7 +275,7 @@ function upsertActivityState(roverId) { return roverActivity.get(roverId); } -function onSensorEvent({ roverId, sensors } = {}) { +function onSensorEvent({ roverId, sensors, batteryState } = {}) { if (!roverId || !sensors) return; const nowMs = Date.now(); const bucketTs = Math.floor(nowMs / ACTIVITY_BUCKET_MS) * ACTIVITY_BUCKET_MS; @@ -286,18 +297,78 @@ function onSensorEvent({ roverId, sensors } = {}) { } state.bumpLeftActive = bumpLeftNow; state.bumpRightActive = bumpRightNow; + + const roverKey = String(roverId); + const activeDrivers = getActiveDrivers(); + const driverSocketId = activeDrivers[roverKey] || null; + const driverNickname = driverSocketId ? resolveDriverNickname(driverSocketId) : null; + const docked = Boolean(sensors?.chargingSources?.homeBase); + const charging = isChargingFromSensors(sensors); + const wheelsOffGround = Boolean( + sensors?.bumpsAndWheelDrops?.wheelDropLeft && sensors?.bumpsAndWheelDrops?.wheelDropRight, + ); + const batteryLow = Boolean(batteryState?.warnActive || batteryState?.urgentActive); + const prevFlags = lastSensorFlagsByRover.get(roverKey) || null; + const nextFlags = { + docked, + charging, + battery_low: batteryLow, + wheels_off_ground: wheelsOffGround, + }; + if (prevFlags) { + if (prevFlags.docked !== nextFlags.docked) { + pushRoverMajorEvent({ + ts: nowMs, + type: 'event', + event_type: nextFlags.docked ? 'rover_docked' : 'rover_undocked', + rover_id: roverKey, + driver_nickname: driverNickname, + summary: nextFlags.docked ? 'transitioned to docked' : 'transitioned to undocked', + }); + } + if (prevFlags.battery_low !== nextFlags.battery_low) { + pushRoverMajorEvent({ + ts: nowMs, + type: 'event', + event_type: 'battery_low_changed', + rover_id: roverKey, + driver_nickname: driverNickname, + summary: nextFlags.battery_low ? 'battery_low became true' : 'battery_low became false', + }); + } + if (prevFlags.wheels_off_ground !== nextFlags.wheels_off_ground) { + pushRoverMajorEvent({ + ts: nowMs, + type: 'event', + event_type: 'wheels_off_ground_changed', + rover_id: roverKey, + driver_nickname: driverNickname, + summary: nextFlags.wheels_off_ground + ? 'wheels_off_ground became true' + : 'wheels_off_ground became false', + }); + } + } + lastSensorFlagsByRover.set(roverKey, nextFlags); } function getActivity30s(roverId, nowMs = Date.now()) { + return getActivityWindow(roverId, ACTIVITY_SCORE_WINDOW_MS, 0, nowMs); +} + +function getActivityWindow(roverId, windowMs, offsetMs = 0, nowMs = Date.now()) { const state = roverActivity.get(String(roverId)); if (!state) { return { distance_m: 0, turn_deg: 0, bumps: 0 }; } pruneActivityBuckets(state, nowMs); + const windowStart = nowMs - offsetMs - windowMs; + const windowEnd = nowMs - offsetMs; let distanceMm = 0; let turnDeg = 0; let bumps = 0; - state.buckets.forEach((bucket) => { + state.buckets.forEach((bucket, bucketTs) => { + if (bucketTs < windowStart || bucketTs > windowEnd) return; distanceMm += bucket.distanceMm; turnDeg += bucket.turnDeg; bumps += bucket.bumps; @@ -309,6 +380,28 @@ function getActivity30s(roverId, nowMs = Date.now()) { }; } +function computeBaseActivityScore(activity = {}) { + const distanceScore = Math.min(45, Math.max(0, Number(activity.distance_m) || 0) * 25); + const turnScore = Math.min(30, Math.max(0, Number(activity.turn_deg) || 0) / 12); + const bumpScore = Math.min(25, Math.max(0, Number(activity.bumps) || 0) * 12); + return Math.round(Math.min(100, distanceScore + turnScore + bumpScore)); +} + +function computeActivityBand(score) { + if (score >= 75) return 'intense'; + if (score >= 50) return 'high'; + if (score >= 25) return 'medium'; + if (score >= 8) return 'low'; + return 'idle'; +} + +function computeActivityTrend(currentBaseScore, previousBaseScore) { + const delta = Number(currentBaseScore || 0) - Number(previousBaseScore || 0); + if (delta >= 12) return 'rising'; + if (delta <= -12) return 'falling'; + return 'steady'; +} + function clearRuntimeHistory() { contextResetAt = Date.now(); clearCount += 1; @@ -317,6 +410,8 @@ function clearRuntimeHistory() { generationTotalMs = 0; runHistory = []; currentRun = null; + roverMajorEvents.length = 0; + lastSensorFlagsByRover.clear(); roverActivity.clear(); lastRoverStateById.clear(); updateStatus({ @@ -473,10 +568,45 @@ function compactRoverForContext(rover) { docked: rover.docked, charging: rover.charging, wheels_off_ground: rover.wheels_off_ground, - activity_30s: rover.activity_30s, + contact_state: rover.contact_state || 'clear', + hazard_state: rover.hazard_state || 'normal', + mobility_state: rover.mobility_state || 'normal', + activity_score: rover.activity_score ?? 0, + activity_band: rover.activity_band || 'idle', + activity_trend: rover.activity_trend || 'steady', }; } +function deriveContactState(sensors = {}, activity30s = {}) { + const bumps = Number(activity30s?.bumps) || 0; + const hasBump = bumps >= 0.5 || sensors?.bumpsAndWheelDrops?.bumpLeft || sensors?.bumpsAndWheelDrops?.bumpRight; + if (hasBump) return 'bumps_recent'; + const light = sensors?.lightBumper || {}; + const wallBrush = + Boolean(sensors?.wall) || + Boolean(light.left || light.frontLeft || light.centerLeft || light.centerRight || light.frontRight || light.right); + if (wallBrush) return 'wall_brush'; + return 'clear'; +} + +function deriveHazardState(sensors = {}) { + if (Boolean(sensors?.virtualWall)) return 'virtual_wall_seen'; + if ( + Boolean(sensors?.cliffLeft) || + Boolean(sensors?.cliffFrontLeft) || + Boolean(sensors?.cliffFrontRight) || + Boolean(sensors?.cliffRight) + ) { + return 'cliff_alert'; + } + return 'normal'; +} + +function deriveMobilityState(sensors = {}, wheelsOffGround = false) { + if (wheelsOffGround) return 'wheels_off_ground'; + return 'normal'; +} + function buildRoversNow(nowMs = Date.now()) { const activeDrivers = getActiveDrivers(); const roster = roverManager.getRoster().slice(0, MAX_ROVERS); @@ -491,6 +621,12 @@ function buildRoversNow(nowMs = Date.now()) { sensors?.bumpsAndWheelDrops?.wheelDropLeft && sensors?.bumpsAndWheelDrops?.wheelDropRight, ); const activity30s = getActivity30s(roverId, nowMs); + const previousActivity30s = getActivityWindow( + roverId, + ACTIVITY_SCORE_WINDOW_MS, + ACTIVITY_SCORE_WINDOW_MS, + nowMs, + ); const charging = isChargingFromSensors(sensors); const docked = Boolean(sensors?.chargingSources?.homeBase); const isMoving = activity30s.distance_m > 0.1 || activity30s.turn_deg > 20; @@ -514,7 +650,24 @@ function buildRoversNow(nowMs = Date.now()) { battery_low: Boolean(batteryState?.warnActive || batteryState?.urgentActive), activity_30s: activity30s, status_tag: statusTag, + contact_state: deriveContactState(sensors, activity30s), + hazard_state: deriveHazardState(sensors), + mobility_state: deriveMobilityState(sensors, wheelsOffGround), }; + const currentBaseScore = computeBaseActivityScore(activity30s); + const previousBaseScore = computeBaseActivityScore(previousActivity30s); + let activityScore = currentBaseScore; + if (rover.contact_state === 'wall_brush') activityScore += 6; + if (rover.contact_state === 'bumps_recent') activityScore += 12; + if (rover.hazard_state !== 'normal') activityScore += 8; + if (rover.status_tag === 'driving') activityScore += 8; + if (rover.status_tag === 'active-idle') activityScore += 4; + if (rover.charging || rover.docked) activityScore -= 25; + if (rover.wheels_off_ground) activityScore -= 20; + activityScore = Math.max(0, Math.min(100, Math.round(activityScore))); + rover.activity_score = activityScore; + rover.activity_band = computeActivityBand(activityScore); + rover.activity_trend = computeActivityTrend(currentBaseScore, previousBaseScore); const prev = lastRoverStateById.get(roverId) || null; const nextState = { driver_nickname: rover.driver_nickname, @@ -524,6 +677,9 @@ function buildRoversNow(nowMs = Date.now()) { battery_low: rover.battery_low, activity_30s: rover.activity_30s, status_tag: rover.status_tag, + activity_score: rover.activity_score, + activity_band: rover.activity_band, + activity_trend: rover.activity_trend, }; nextRoverStateById.set(roverId, nextState); rover.prev_state = prev; @@ -560,7 +716,24 @@ function buildSnapshot() { (entry) => nowMs - Number(entry?.ts || 0) <= SELF_TALK_WINDOW_MS, ); - const eventStream = allRecentMessages.slice(-MAX_CONTEXT_EVENTS).map((entry) => { + const roverEvents = roverMajorEvents.filter((entry) => Number(entry?.ts) >= contextResetAt); + const timelineEntries = [ + ...allRecentMessages.map((entry) => ({ ts: Number(entry?.ts || 0), source: 'chat', entry })), + ...roverEvents.map((entry) => ({ ts: Number(entry?.ts || 0), source: 'event', entry })), + ] + .sort((a, b) => a.ts - b.ts) + .slice(-MAX_CONTEXT_EVENTS); + + const eventStream = timelineEntries.map(({ source, entry }) => { + if (source === 'event') { + return { + type: 'event', + event_type: entry.event_type || 'rover_event', + rover_id: entry.rover_id || null, + driver_nickname: entry.driver_nickname || null, + summary: entry.summary || '', + }; + } if (entry?.system) { return { type: 'bot', @@ -570,12 +743,14 @@ function buildSnapshot() { } const roverId = entry?.roverId ? String(entry.roverId) : null; const rover = roverId ? roverById.get(roverId) : null; + const baseCtx = compactRoverForContext(rover) || {}; + const storedCtx = entry?.roverCtx || entry?.rover_ctx || {}; return { type: 'chat', nickname: entry.nickname || entry.socketId?.slice(0, 6) || 'unknown', text: entry.text || '', rover_id: roverId, - rover_ctx: entry?.roverCtx || entry?.rover_ctx || compactRoverForContext(rover), + rover_ctx: { ...baseCtx, ...storedCtx }, }; }); const hasRecentChat = eventStream.some((event) => event.type === 'chat'); @@ -663,8 +838,7 @@ function formatRunMetaMessage(runMeta = {}) { function formatRoverCtx(ctx) { if (!ctx || typeof ctx !== 'object') return 'none'; - const activity = ctx.activity_30s || {}; - return `status=${ctx.status_tag || 'unknown'} battery_low=${Boolean(ctx.battery_low)} wheels_off_ground=${Boolean(ctx.wheels_off_ground)} docked=${Boolean(ctx.docked)} charging=${Boolean(ctx.charging)} move30=${Number(activity.distance_m) || 0}m turn30=${Number(activity.turn_deg) || 0}deg bumps30=${Number(activity.bumps) || 0}`; + return `status=${ctx.status_tag || 'unknown'} battery_low=${Boolean(ctx.battery_low)} docked=${Boolean(ctx.docked)} charging=${Boolean(ctx.charging)} wheels_off_ground=${Boolean(ctx.wheels_off_ground)} contact=${ctx.contact_state || 'clear'} hazard=${ctx.hazard_state || 'normal'} mobility=${ctx.mobility_state || 'normal'} activity_score=${Number(ctx.activity_score) || 0} activity_band=${ctx.activity_band || 'idle'} activity_trend=${ctx.activity_trend || 'steady'}`; } function formatChatEventMessage(event) { @@ -678,8 +852,17 @@ function formatChatEventMessage(event) { } function formatRoverSnapshotLine(rover = {}) { - const activity = rover.activity_30s || {}; - return `${rover.id || 'unknown'}: driver=${rover.driver_nickname || 'none'} status=${rover.status_tag || 'unknown'} battery_low=${Boolean(rover.battery_low)} wheels_off_ground=${Boolean(rover.wheels_off_ground)} docked=${Boolean(rover.docked)} charging=${Boolean(rover.charging)} move30=${Number(activity.distance_m) || 0}m turn30=${Number(activity.turn_deg) || 0}deg bumps30=${Number(activity.bumps) || 0}`; + return `${rover.id || 'unknown'}: driver=${rover.driver_nickname || 'none'} status=${rover.status_tag || 'unknown'} battery_low=${Boolean(rover.battery_low)} docked=${Boolean(rover.docked)} charging=${Boolean(rover.charging)} wheels_off_ground=${Boolean(rover.wheels_off_ground)} contact=${rover.contact_state || 'clear'} hazard=${rover.hazard_state || 'normal'} mobility=${rover.mobility_state || 'normal'} activity_score=${Number(rover.activity_score) || 0} activity_band=${rover.activity_band || 'idle'} activity_trend=${rover.activity_trend || 'steady'}`; +} + +function formatEventMessage(event) { + return [ + 'EVENT', + `type: ${event.event_type || 'rover_event'}`, + `rover: ${event.rover_id || 'unknown'}`, + `driver: ${event.driver_nickname || 'none'}`, + `summary: ${event.summary || ''}`, + ].join('\n'); } function formatSnapshotMessage(event) { @@ -744,6 +927,13 @@ function buildModelMessages(systemPrompt, snapshot) { }); return; } + if (event.type === 'event') { + messages.push({ + role: 'user', + content: formatEventMessage(event), + }); + return; + } if (event.type === 'snapshot') { messages.push({ role: 'user',