legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-16 01:21:20 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

here goes nothin

Browse Source
This commit is contained in:
legop3
2025-11-29 21:24:18 -05:00
parent d00b2cf160
commit f6a57b788a
13 changed files with 247 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi/bin/audio-only-publisher.sh
+5
View File
@@ -11,6 +11,11 @@ fi
# shellcheck disable=SC1090
source "$ENV_FILE"
AUDIO_ENABLE="${AUDIO_ENABLE:-0}"
if [[ "${AUDIO_ENABLE}" -ne 1 ]]; then
echo "Audio capture disabled; skipping audio-only publisher" >&2
exit 0
fi
: "${AUDIO_PUBLISH_URL:?AUDIO_PUBLISH_URL not set in ${ENV_FILE}}"
AUDIO_DEVICE="${AUDIO_DEVICE:-hw:0,0}"
pi/systemd/audio-only-publisher.service
+1 -1
View File
@@ -9,7 +9,7 @@ User=roverd
Group=roverd
EnvironmentFile=/var/lib/roverd/video.env
ExecStart=/usr/local/bin/audio-only-publisher
Restart=always
Restart=on-failure
RestartSec=2
[Install]
server/public/assets/index-Ch0OrDUK.js
-11
View File
File diff suppressed because one or more lines are too long
server/public/assets/index-DdtxDvI9.js
+11
View File
File diff suppressed because one or more lines are too long
server/public/index.html
+1 -1
View File
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="apple-mobile-web-app-title" content="Multi Roomba Rover" />
<title>Multi Roomba Rover</title>
<script type="module" crossorigin src="/assets/index-Ch0OrDUK.js"></script>
<script type="module" crossorigin src="/assets/index-DdtxDvI9.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BxUpjkzh.css">
</head>
<body>
server/src/services/videoAuthService.js
+3 -1
View File
@@ -45,7 +45,9 @@ function extractStreamInfo(path) {
const remaining = segments.slice(start, end);
if (remaining.length === 1) {
return { type: 'rover', id: remaining[0] || '' };
const rawId = remaining[0] || '';
const id = rawId.endsWith('-audio') ? rawId.slice(0, -6) : rawId;
return { type: 'rover', id };
}
if (remaining.length === 2 && remaining[0] === 'room') {
return { type: 'room', id: remaining[1] || '' };
webui/src/components/DriverVideoPanel.jsx
+15 -2
View File
@@ -6,8 +6,14 @@ import VideoTile from './VideoTile.jsx';
export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
const { session } = useSession();
const roverId = session?.assignment?.roverId;
const sources = useVideoRequests(roverId ? [roverId] : []);
const rosterEntry =
roverId && session?.roster ? session.roster.find((item) => String(item.id) === String(roverId)) : null;
const hasAudio = Boolean(rosterEntry?.media?.audioPublishUrl);
const sources = useVideoRequests(
roverId ? [{ type: 'rover', id: roverId, audioId: hasAudio ? `${roverId}-audio` : null }] : [],
);
const info = roverId ? sources[roverId] : null;
const audioInfo = roverId && hasAudio ? sources[`${roverId}-audio`] : null;
const frame = useTelemetryFrame(roverId);
const batteryRecord =
roverId && session?.roster
@@ -20,7 +26,14 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
return (
<section className="panel">
{roverId ? (
<VideoTile sessionInfo={info} label={roverLabel} telemetryFrame={frame} batteryConfig={batteryConfig} layoutFormat={layoutFormat}/>
<VideoTile
sessionInfo={info}
audioSessionInfo={audioInfo}
label={roverLabel}
telemetryFrame={frame}
batteryConfig={batteryConfig}
layoutFormat={layoutFormat}
/>
) : (
<div className="panel-muted content-center text-center text-sm text-slate-400 aspect-video">
<p>You are not assigned to a rover.</p>
webui/src/components/VideoTile.jsx
+52
View File
@@ -1,11 +1,13 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { WhepPlayer } from '../lib/whepPlayer.js';
import { AudioWhepPlayer } from '../lib/audioWhepPlayer.js';
const RESTART_DELAY_MS = 2000;
const UNMUTE_RETRY_MS = 3000;
export default function VideoTile({
sessionInfo,
audioSessionInfo,
label,
forceMute = false,
telemetryFrame,
@@ -13,11 +15,16 @@ export default function VideoTile({
layoutFormat = 'desktop',
}) {
const videoRef = useRef(null);
const audioRef = useRef(null);
const restartTimer = useRef(null);
const audioRestartTimer = useRef(null);
const unmuteTimer = useRef(null);
const [status, setStatus] = useState('idle');
const [detail, setDetail] = useState(null);
const [audioStatus, setAudioStatus] = useState('idle');
const [audioDetail, setAudioDetail] = useState(null);
const [restartToken, setRestartToken] = useState(0);
const [audioRestartToken, setAudioRestartToken] = useState(0);
const [muted, setMuted] = useState(true);
const sensors = telemetryFrame?.sensors;
const batteryCharge = sensors?.batteryChargeMah ?? null;
@@ -40,6 +47,10 @@ export default function VideoTile({
clearTimeout(restartTimer.current);
restartTimer.current = setTimeout(() => setRestartToken(Date.now()), RESTART_DELAY_MS);
}, []);
const scheduleAudioRestart = useCallback(() => {
clearTimeout(audioRestartTimer.current);
audioRestartTimer.current = setTimeout(() => setAudioRestartToken(Date.now()), RESTART_DELAY_MS);
}, []);
const ensurePlayback = useCallback(async () => {
const video = videoRef.current;
@@ -89,6 +100,7 @@ export default function VideoTile({
useEffect(
() => () => {
clearTimeout(restartTimer.current);
clearTimeout(audioRestartTimer.current);
clearTimeout(unmuteTimer.current);
},
[],
@@ -146,6 +158,45 @@ export default function VideoTile({
}
}, [status, sessionInfo?.url, scheduleRestart]);
// Audio-only WHEP (no pausing/muting; keeps trying to play)
useEffect(() => {
if (!audioSessionInfo?.url || !audioRef.current) {
return undefined;
}
let active = true;
let player;
const handleStatus = (nextStatus, info) => {
if (!active) return;
setAudioStatus(nextStatus);
setAudioDetail(info || null);
if (nextStatus === 'playing') {
audioRef.current?.play().catch(() => {});
}
if (['error', 'failed', 'disconnected', 'closed'].includes(nextStatus)) {
scheduleAudioRestart();
}
};
player = new AudioWhepPlayer({
url: audioSessionInfo.url,
token: audioSessionInfo.token,
audio: audioRef.current,
onStatus: handleStatus,
});
player.start().catch((err) => {
if (!active) return;
setAudioStatus('error');
setAudioDetail(err.message);
scheduleAudioRestart();
});
return () => {
active = false;
player?.stop();
};
}, [audioSessionInfo?.url, audioSessionInfo?.token, audioRestartToken, scheduleAudioRestart]);
const renderedStatus = !sessionInfo?.url
? 'waiting'
: status === 'error'
@@ -165,6 +216,7 @@ export default function VideoTile({
controls={false}
className="h-full w-full object-contain"
/>
<audio ref={audioRef} autoPlay hidden />
<HudOverlay frame={telemetryFrame} label={label} status={renderedStatus} desktopLayout={desktopLayout}/>
<OvercurrentOverlay motors={overcurrentMotors} />
<LowBatteryOverlay charge={batteryCharge} config={batteryConfig} />
webui/src/hooks/useVideoRequests.js
+17 -5
View File
@@ -13,7 +13,12 @@ function normalizeEntry(entry) {
if (typeof entry === 'object') {
if (entry.type && entry.id) {
const id = String(entry.id);
return { type: entry.type, id, key: entry.key || `${entry.type}:${id}` };
return {
type: entry.type,
id,
key: entry.key || `${entry.type}:${id}`,
audioId: entry.audioId ? String(entry.audioId) : null,
};
}
if (entry.roverId) {
const id = String(entry.roverId);
@@ -99,10 +104,17 @@ export function useVideoRequests(sourceList = []) {
return;
}
clearRetry(entry.key);
setSources((prev) => ({
...prev,
[entry.key]: resp,
}));
setSources((prev) => {
const next = { ...prev, [entry.key]: resp };
if (entry.audioId && resp.url && resp.token) {
const audioUrl = resp.url.replace(
`/${encodeURIComponent(entry.id)}/whep`,
`/${encodeURIComponent(entry.audioId)}/whep`,
);
next[entry.audioId] = { url: audioUrl, token: resp.token };
}
return next;
});
});
}
webui/src/lib/audioWhepPlayer.js
+114
View File
@@ -0,0 +1,114 @@
/* global Buffer */
import { buildAuthHeader } from './whepAuth.js';
const RTC_CONFIG = {
iceServers: [
{ urls: 'stun:stun.l.google.com:19302' },
{ urls: 'turn:your.turn.server:3478', username: 'user', credential: 'pass' },
],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
export class AudioWhepPlayer {
constructor({ url, token, audio, onStatus }) {
this.url = url;
this.token = token;
this.audio = audio;
this.pc = null;
this.abortController = null;
this.onStatus = onStatus;
}
notify(status, detail) {
if (typeof this.onStatus === 'function') {
this.onStatus(status, detail);
}
}
configureElement() {
if (!this.audio) return;
this.audio.autoplay = true;
this.audio.muted = false;
this.audio.playsInline = true;
}
async start() {
if (!this.url || !this.audio) {
throw new Error('Audio target missing');
}
this.stop();
this.configureElement();
this.notify('connecting');
this.abortController = new AbortController();
const pc = new RTCPeerConnection(RTC_CONFIG);
this.pc = pc;
const stream = new MediaStream();
pc.ontrack = (event) => {
event.streams[0]?.getTracks().forEach((track) => stream.addTrack(track));
this.audio.srcObject = stream;
if (event.receiver && 'playoutDelayHint' in event.receiver) {
event.receiver.playoutDelayHint = 0;
}
};
pc.addTransceiver('audio', { direction: 'recvonly' });
pc.onconnectionstatechange = () => {
this.notify(pc.connectionState);
};
pc.oniceconnectionstatechange = () => {
const state = pc.iceConnectionState;
if (state === 'failed' || state === 'disconnected') {
this.notify(state);
}
};
try {
const offer = await pc.createOffer({ offerToReceiveAudio: true, offerToReceiveVideo: false });
await pc.setLocalDescription(offer);
const response = await fetch(this.url, {
method: 'POST',
headers: {
'Content-Type': 'application/sdp',
...buildAuthHeader(this.token),
},
body: offer.sdp,
signal: this.abortController.signal,
});
if (!response.ok) {
throw new Error(`WHEP audio request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
pc.getReceivers().forEach((receiver) => {
if ('playoutDelayHint' in receiver) {
receiver.playoutDelayHint = 0;
}
});
await this.audio.play().catch(() => {});
this.notify('playing');
} catch (err) {
this.notify('error', err.message);
this.stop();
throw err;
}
}
stop() {
if (this.abortController) {
this.abortController.abort();
this.abortController = null;
}
if (this.pc) {
this.pc.getSenders().forEach((s) => s.track?.stop());
this.pc.getReceivers().forEach((r) => r.track?.stop());
this.pc.close();
this.pc = null;
}
if (this.audio) {
this.audio.srcObject = null;
}
this.notify('stopped');
}
}
webui/src/lib/whepAuth.js
+18
View File
@@ -0,0 +1,18 @@
/* global Buffer */
function encodeBase64(value) {
if (typeof btoa === 'function') {
return btoa(value);
}
if (typeof Buffer !== 'undefined') {
return Buffer.from(value).toString('base64');
}
throw new Error('No base64 encoder available');
}
export function buildAuthHeader(token) {
if (!token) return {};
const credential = `${token}:${token}`;
const encoded = encodeBase64(credential);
return { Authorization: `Basic ${encoded}` };
}
webui/src/lib/whepPlayer.js
+1 -19
View File
@@ -1,4 +1,4 @@
/* global Buffer */
import { buildAuthHeader } from './whepAuth.js';
const RTC_CONFIG = {
iceServers: [
@@ -9,24 +9,6 @@ const RTC_CONFIG = {
rtcpMuxPolicy: 'require',
};
function encodeBase64(value) {
if (typeof btoa === 'function') {
return btoa(value);
}
if (typeof Buffer !== 'undefined') {
return Buffer.from(value).toString('base64');
}
throw new Error('No base64 encoder available');
}
function buildAuthHeader(token) {
if (!token) return {};
const credential = `${token}:${token}`;
const encoded = encodeBase64(credential);
return { Authorization: `Basic ${encoded}` };
}
export class WhepPlayer {
constructor({ url, token, video, onStatus }) {
this.url = url;
webui/src/spectate/SpectatorApp.jsx
+9 -2
View File
@@ -49,7 +49,7 @@ function CurrentDriverBadge({ roverId, session }) {
);
}
function RoverSpectatorCard({ rover, frame, videoInfo, session }) {
function RoverSpectatorCard({ rover, frame, videoInfo, audioInfo, session }) {
return (
<article className="grid min-h-[16rem] grid-rows-[auto_minmax(0,1fr)_auto] gap-0.5 rounded bg-zinc-900 p-0.5 sm:min-h-[18rem]">
<header className="flex items-center justify-between gap-0.5">
@@ -64,6 +64,7 @@ function RoverSpectatorCard({ rover, frame, videoInfo, session }) {
<div className="min-h-0 overflow-hidden rounded bg-black/20">
<VideoTile
sessionInfo={videoInfo}
audioSessionInfo={audioInfo}
label={rover.name}
telemetryFrame={frame}
batteryConfig={rover.battery}
@@ -86,6 +87,7 @@ function RoverRow({ roster, frames, videoSources, session }) {
rover={rover}
frame={frames[rover.id]}
videoInfo={videoSources[rover.id]}
audioInfo={videoSources[`${rover.id}-audio`]}
session={session}
/>
))}
@@ -116,7 +118,12 @@ export default function SpectatorApp() {
useSpectatorMode();
const frames = useTelemetryFrames();
const roster = session?.roster ?? [];
const videoSources = useVideoRequests(roster.map((rover) => rover.id));
const entries = roster.map((rover) => ({
type: 'rover',
id: rover.id,
audioId: rover.media?.audioPublishUrl ? `${rover.id}-audio` : null,
}));
const videoSources = useVideoRequests(entries);
return (
<SettingsProvider>