legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-17 10:00:46 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

here goes nothin

Browse Source
This commit is contained in:
legop3
2025-11-29 21:24:18 -05:00
parent d00b2cf160
commit f6a57b788a
13 changed files with 247 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi/bin/audio-only-publisher.sh
+5
View File
@@ -11,6 +11,11 @@ fi
# shellcheck disable=SC1090 # shellcheck disable=SC1090
source "$ENV_FILE" source "$ENV_FILE"
AUDIO_ENABLE="${AUDIO_ENABLE:-0}"
if [[ "${AUDIO_ENABLE}" -ne 1 ]]; then
echo "Audio capture disabled; skipping audio-only publisher" >&2
exit 0
fi
: "${AUDIO_PUBLISH_URL:?AUDIO_PUBLISH_URL not set in ${ENV_FILE}}" : "${AUDIO_PUBLISH_URL:?AUDIO_PUBLISH_URL not set in ${ENV_FILE}}"
AUDIO_DEVICE="${AUDIO_DEVICE:-hw:0,0}" AUDIO_DEVICE="${AUDIO_DEVICE:-hw:0,0}"
pi/systemd/audio-only-publisher.service
+1 -1
View File
@@ -9,7 +9,7 @@ User=roverd
Group=roverd Group=roverd
EnvironmentFile=/var/lib/roverd/video.env EnvironmentFile=/var/lib/roverd/video.env
ExecStart=/usr/local/bin/audio-only-publisher ExecStart=/usr/local/bin/audio-only-publisher
Restart=always Restart=on-failure
RestartSec=2 RestartSec=2
[Install] [Install]
server/public/assets/index-Ch0OrDUK.js
-11
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-DdtxDvI9.js
+11
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" /> <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" /> <meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title> <title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-Ch0OrDUK.js"></script> <script type="module" crossorigin src="/assets/index-DdtxDvI9.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BxUpjkzh.css"> <link rel="stylesheet" crossorigin href="/assets/index-BxUpjkzh.css">
</head> </head>
<body> <body>
server/src/services/videoAuthService.js
+3 -1
View File
@@ -45,7 +45,9 @@ function extractStreamInfo(path) {
const remaining = segments.slice(start, end); const remaining = segments.slice(start, end);
if (remaining.length === 1) { if (remaining.length === 1) {
return { type: 'rover', id: remaining[0] || '' }; const rawId = remaining[0] || '';
const id = rawId.endsWith('-audio') ? rawId.slice(0, -6) : rawId;
return { type: 'rover', id };
} }
if (remaining.length === 2 && remaining[0] === 'room') { if (remaining.length === 2 && remaining[0] === 'room') {
return { type: 'room', id: remaining[1] || '' }; return { type: 'room', id: remaining[1] || '' };
webui/src/components/DriverVideoPanel.jsx
+15 -2
View File
@@ -6,8 +6,14 @@ import VideoTile from './VideoTile.jsx';
export default function DriverVideoPanel({layoutFormat = 'desktop'}) { export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
const { session } = useSession(); const { session } = useSession();
const roverId = session?.assignment?.roverId; const roverId = session?.assignment?.roverId;
const sources = useVideoRequests(roverId ? [roverId] : []); const rosterEntry =
roverId && session?.roster ? session.roster.find((item) => String(item.id) === String(roverId)) : null;
const hasAudio = Boolean(rosterEntry?.media?.audioPublishUrl);
const sources = useVideoRequests(
roverId ? [{ type: 'rover', id: roverId, audioId: hasAudio ? `${roverId}-audio` : null }] : [],
);
const info = roverId ? sources[roverId] : null; const info = roverId ? sources[roverId] : null;
const audioInfo = roverId && hasAudio ? sources[`${roverId}-audio`] : null;
const frame = useTelemetryFrame(roverId); const frame = useTelemetryFrame(roverId);
const batteryRecord = const batteryRecord =
roverId && session?.roster roverId && session?.roster
@@ -20,7 +26,14 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
return ( return (
<section className="panel"> <section className="panel">
{roverId ? ( {roverId ? (
<VideoTile sessionInfo={info} label={roverLabel} telemetryFrame={frame} batteryConfig={batteryConfig} layoutFormat={layoutFormat}/> <VideoTile
sessionInfo={info}
audioSessionInfo={audioInfo}
label={roverLabel}
telemetryFrame={frame}
batteryConfig={batteryConfig}
layoutFormat={layoutFormat}
/>
) : ( ) : (
<div className="panel-muted content-center text-center text-sm text-slate-400 aspect-video"> <div className="panel-muted content-center text-center text-sm text-slate-400 aspect-video">
<p>You are not assigned to a rover.</p> <p>You are not assigned to a rover.</p>
webui/src/components/VideoTile.jsx
+52
View File
@@ -1,11 +1,13 @@
import { useCallback, useEffect, useRef, useState } from 'react'; import { useCallback, useEffect, useRef, useState } from 'react';
import { WhepPlayer } from '../lib/whepPlayer.js'; import { WhepPlayer } from '../lib/whepPlayer.js';
import { AudioWhepPlayer } from '../lib/audioWhepPlayer.js';
const RESTART_DELAY_MS = 2000; const RESTART_DELAY_MS = 2000;
const UNMUTE_RETRY_MS = 3000; const UNMUTE_RETRY_MS = 3000;
export default function VideoTile({ export default function VideoTile({
sessionInfo, sessionInfo,
audioSessionInfo,
label, label,
forceMute = false, forceMute = false,
telemetryFrame, telemetryFrame,
@@ -13,11 +15,16 @@ export default function VideoTile({
layoutFormat = 'desktop', layoutFormat = 'desktop',
}) { }) {
const videoRef = useRef(null); const videoRef = useRef(null);
const audioRef = useRef(null);
const restartTimer = useRef(null); const restartTimer = useRef(null);
const audioRestartTimer = useRef(null);
const unmuteTimer = useRef(null); const unmuteTimer = useRef(null);
const [status, setStatus] = useState('idle'); const [status, setStatus] = useState('idle');
const [detail, setDetail] = useState(null); const [detail, setDetail] = useState(null);
const [audioStatus, setAudioStatus] = useState('idle');
const [audioDetail, setAudioDetail] = useState(null);
const [restartToken, setRestartToken] = useState(0); const [restartToken, setRestartToken] = useState(0);
const [audioRestartToken, setAudioRestartToken] = useState(0);
const [muted, setMuted] = useState(true); const [muted, setMuted] = useState(true);
const sensors = telemetryFrame?.sensors; const sensors = telemetryFrame?.sensors;
const batteryCharge = sensors?.batteryChargeMah ?? null; const batteryCharge = sensors?.batteryChargeMah ?? null;
@@ -40,6 +47,10 @@ export default function VideoTile({
clearTimeout(restartTimer.current); clearTimeout(restartTimer.current);
restartTimer.current = setTimeout(() => setRestartToken(Date.now()), RESTART_DELAY_MS); restartTimer.current = setTimeout(() => setRestartToken(Date.now()), RESTART_DELAY_MS);
}, []); }, []);
const scheduleAudioRestart = useCallback(() => {
clearTimeout(audioRestartTimer.current);
audioRestartTimer.current = setTimeout(() => setAudioRestartToken(Date.now()), RESTART_DELAY_MS);
}, []);
const ensurePlayback = useCallback(async () => { const ensurePlayback = useCallback(async () => {
const video = videoRef.current; const video = videoRef.current;
@@ -89,6 +100,7 @@ export default function VideoTile({
useEffect( useEffect(
() => () => { () => () => {
clearTimeout(restartTimer.current); clearTimeout(restartTimer.current);
clearTimeout(audioRestartTimer.current);
clearTimeout(unmuteTimer.current); clearTimeout(unmuteTimer.current);
}, },
[], [],
@@ -146,6 +158,45 @@ export default function VideoTile({
} }
}, [status, sessionInfo?.url, scheduleRestart]); }, [status, sessionInfo?.url, scheduleRestart]);
// Audio-only WHEP (no pausing/muting; keeps trying to play)
useEffect(() => {
if (!audioSessionInfo?.url || !audioRef.current) {
return undefined;
}
let active = true;
let player;
const handleStatus = (nextStatus, info) => {
if (!active) return;
setAudioStatus(nextStatus);
setAudioDetail(info || null);
if (nextStatus === 'playing') {
audioRef.current?.play().catch(() => {});
}
if (['error', 'failed', 'disconnected', 'closed'].includes(nextStatus)) {
scheduleAudioRestart();
}
};
player = new AudioWhepPlayer({
url: audioSessionInfo.url,
token: audioSessionInfo.token,
audio: audioRef.current,
onStatus: handleStatus,
});
player.start().catch((err) => {
if (!active) return;
setAudioStatus('error');
setAudioDetail(err.message);
scheduleAudioRestart();
});
return () => {
active = false;
player?.stop();
};
}, [audioSessionInfo?.url, audioSessionInfo?.token, audioRestartToken, scheduleAudioRestart]);
const renderedStatus = !sessionInfo?.url const renderedStatus = !sessionInfo?.url
? 'waiting' ? 'waiting'
: status === 'error' : status === 'error'
@@ -165,6 +216,7 @@ export default function VideoTile({
controls={false} controls={false}
className="h-full w-full object-contain" className="h-full w-full object-contain"
/> />
<audio ref={audioRef} autoPlay hidden />
<HudOverlay frame={telemetryFrame} label={label} status={renderedStatus} desktopLayout={desktopLayout}/> <HudOverlay frame={telemetryFrame} label={label} status={renderedStatus} desktopLayout={desktopLayout}/>
<OvercurrentOverlay motors={overcurrentMotors} /> <OvercurrentOverlay motors={overcurrentMotors} />
<LowBatteryOverlay charge={batteryCharge} config={batteryConfig} /> <LowBatteryOverlay charge={batteryCharge} config={batteryConfig} />
webui/src/hooks/useVideoRequests.js
+17 -5
View File
@@ -13,7 +13,12 @@ function normalizeEntry(entry) {
if (typeof entry === 'object') { if (typeof entry === 'object') {
if (entry.type && entry.id) { if (entry.type && entry.id) {
const id = String(entry.id); const id = String(entry.id);
return { type: entry.type, id, key: entry.key || `${entry.type}:${id}` }; return {
type: entry.type,
id,
key: entry.key || `${entry.type}:${id}`,
audioId: entry.audioId ? String(entry.audioId) : null,
};
} }
if (entry.roverId) { if (entry.roverId) {
const id = String(entry.roverId); const id = String(entry.roverId);
@@ -99,10 +104,17 @@ export function useVideoRequests(sourceList = []) {
return; return;
} }
clearRetry(entry.key); clearRetry(entry.key);
setSources((prev) => ({ setSources((prev) => {
...prev, const next = { ...prev, [entry.key]: resp };
[entry.key]: resp, if (entry.audioId && resp.url && resp.token) {
})); const audioUrl = resp.url.replace(
`/${encodeURIComponent(entry.id)}/whep`,
`/${encodeURIComponent(entry.audioId)}/whep`,
);
next[entry.audioId] = { url: audioUrl, token: resp.token };
}
return next;
});
}); });
} }
webui/src/lib/audioWhepPlayer.js
+114
View File
@@ -0,0 +1,114 @@
/* global Buffer */
import { buildAuthHeader } from './whepAuth.js';
const RTC_CONFIG = {
iceServers: [
{ urls: 'stun:stun.l.google.com:19302' },
{ urls: 'turn:your.turn.server:3478', username: 'user', credential: 'pass' },
],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
export class AudioWhepPlayer {
constructor({ url, token, audio, onStatus }) {
this.url = url;
this.token = token;
this.audio = audio;
this.pc = null;
this.abortController = null;
this.onStatus = onStatus;
}
notify(status, detail) {
if (typeof this.onStatus === 'function') {
this.onStatus(status, detail);
}
}
configureElement() {
if (!this.audio) return;
this.audio.autoplay = true;
this.audio.muted = false;
this.audio.playsInline = true;
}
async start() {
if (!this.url || !this.audio) {
throw new Error('Audio target missing');
}
this.stop();
this.configureElement();
this.notify('connecting');
this.abortController = new AbortController();
const pc = new RTCPeerConnection(RTC_CONFIG);
this.pc = pc;
const stream = new MediaStream();
pc.ontrack = (event) => {
event.streams[0]?.getTracks().forEach((track) => stream.addTrack(track));
this.audio.srcObject = stream;
if (event.receiver && 'playoutDelayHint' in event.receiver) {
event.receiver.playoutDelayHint = 0;
}
};
pc.addTransceiver('audio', { direction: 'recvonly' });
pc.onconnectionstatechange = () => {
this.notify(pc.connectionState);
};
pc.oniceconnectionstatechange = () => {
const state = pc.iceConnectionState;
if (state === 'failed' || state === 'disconnected') {
this.notify(state);
}
};
try {
const offer = await pc.createOffer({ offerToReceiveAudio: true, offerToReceiveVideo: false });
await pc.setLocalDescription(offer);
const response = await fetch(this.url, {
method: 'POST',
headers: {
'Content-Type': 'application/sdp',
...buildAuthHeader(this.token),
},
body: offer.sdp,
signal: this.abortController.signal,
});
if (!response.ok) {
throw new Error(`WHEP audio request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
pc.getReceivers().forEach((receiver) => {
if ('playoutDelayHint' in receiver) {
receiver.playoutDelayHint = 0;
}
});
await this.audio.play().catch(() => {});
this.notify('playing');
} catch (err) {
this.notify('error', err.message);
this.stop();
throw err;
}
}
stop() {
if (this.abortController) {
this.abortController.abort();
this.abortController = null;
}
if (this.pc) {
this.pc.getSenders().forEach((s) => s.track?.stop());
this.pc.getReceivers().forEach((r) => r.track?.stop());
this.pc.close();
this.pc = null;
}
if (this.audio) {
this.audio.srcObject = null;
}
this.notify('stopped');
}
}
webui/src/lib/whepAuth.js
+18
View File
@@ -0,0 +1,18 @@
/* global Buffer */
function encodeBase64(value) {
if (typeof btoa === 'function') {
return btoa(value);
}
if (typeof Buffer !== 'undefined') {
return Buffer.from(value).toString('base64');
}
throw new Error('No base64 encoder available');
}
export function buildAuthHeader(token) {
if (!token) return {};
const credential = `${token}:${token}`;
const encoded = encodeBase64(credential);
return { Authorization: `Basic ${encoded}` };
}
webui/src/lib/whepPlayer.js
+1 -19
View File
@@ -1,4 +1,4 @@
/* global Buffer */ import { buildAuthHeader } from './whepAuth.js';
const RTC_CONFIG = { const RTC_CONFIG = {
iceServers: [ iceServers: [
@@ -9,24 +9,6 @@ const RTC_CONFIG = {
rtcpMuxPolicy: 'require', rtcpMuxPolicy: 'require',
}; };
function encodeBase64(value) {
if (typeof btoa === 'function') {
return btoa(value);
}
if (typeof Buffer !== 'undefined') {
return Buffer.from(value).toString('base64');
}
throw new Error('No base64 encoder available');
}
function buildAuthHeader(token) {
if (!token) return {};
const credential = `${token}:${token}`;
const encoded = encodeBase64(credential);
return { Authorization: `Basic ${encoded}` };
}
export class WhepPlayer { export class WhepPlayer {
constructor({ url, token, video, onStatus }) { constructor({ url, token, video, onStatus }) {
this.url = url; this.url = url;
webui/src/spectate/SpectatorApp.jsx
+9 -2
View File
@@ -49,7 +49,7 @@ function CurrentDriverBadge({ roverId, session }) {
); );
} }
function RoverSpectatorCard({ rover, frame, videoInfo, session }) { function RoverSpectatorCard({ rover, frame, videoInfo, audioInfo, session }) {
return ( return (
<article className="grid min-h-[16rem] grid-rows-[auto_minmax(0,1fr)_auto] gap-0.5 rounded bg-zinc-900 p-0.5 sm:min-h-[18rem]"> <article className="grid min-h-[16rem] grid-rows-[auto_minmax(0,1fr)_auto] gap-0.5 rounded bg-zinc-900 p-0.5 sm:min-h-[18rem]">
<header className="flex items-center justify-between gap-0.5"> <header className="flex items-center justify-between gap-0.5">
@@ -64,6 +64,7 @@ function RoverSpectatorCard({ rover, frame, videoInfo, session }) {
<div className="min-h-0 overflow-hidden rounded bg-black/20"> <div className="min-h-0 overflow-hidden rounded bg-black/20">
<VideoTile <VideoTile
sessionInfo={videoInfo} sessionInfo={videoInfo}
audioSessionInfo={audioInfo}
label={rover.name} label={rover.name}
telemetryFrame={frame} telemetryFrame={frame}
batteryConfig={rover.battery} batteryConfig={rover.battery}
@@ -86,6 +87,7 @@ function RoverRow({ roster, frames, videoSources, session }) {
rover={rover} rover={rover}
frame={frames[rover.id]} frame={frames[rover.id]}
videoInfo={videoSources[rover.id]} videoInfo={videoSources[rover.id]}
audioInfo={videoSources[`${rover.id}-audio`]}
session={session} session={session}
/> />
))} ))}
@@ -116,7 +118,12 @@ export default function SpectatorApp() {
useSpectatorMode(); useSpectatorMode();
const frames = useTelemetryFrames(); const frames = useTelemetryFrames();
const roster = session?.roster ?? []; const roster = session?.roster ?? [];
const videoSources = useVideoRequests(roster.map((rover) => rover.id)); const entries = roster.map((rover) => ({
type: 'rover',
id: rover.id,
audioId: rover.media?.audioPublishUrl ? `${rover.id}-audio` : null,
}));
const videoSources = useVideoRequests(entries);
return ( return (
<SettingsProvider> <SettingsProvider>