import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { fieldClass } from './constants.js';
import { useControlSystem } from '../../controls/index.js';
import { useSettingsNamespace } from '../../settings/index.js';
const MAX_UPLOAD_BYTES = 8 * 1024 * 1024;
const TARGET_SAMPLE_RATE = 16000;
const RTC_CONFIG = {
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
bundlePolicy: 'max-bundle',
rtcpMuxPolicy: 'require',
};
function bytesToBase64(bytes) {
let binary = '';
const chunkSize = 0x8000;
for (let i = 0; i < bytes.length; i += chunkSize) {
const chunk = bytes.subarray(i, i + chunkSize);
binary += String.fromCharCode(...chunk);
}
return btoa(binary);
}
function encodeBase64(value) {
if (typeof btoa === 'function') return btoa(value);
return '';
}
function buildAuthHeader(token) {
if (!token) return {};
const encoded = encodeBase64(`${token}:${token}`);
return encoded ? { Authorization: `Basic ${encoded}` } : {};
}
function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
return new Promise((resolve) => {
if (!pc || pc.iceGatheringState === 'complete') {
resolve();
return;
}
const timer = setTimeout(() => {
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}, timeoutMs);
function onChange() {
if (pc.iceGatheringState === 'complete') {
clearTimeout(timer);
pc.removeEventListener('icegatheringstatechange', onChange);
resolve();
}
}
pc.addEventListener('icegatheringstatechange', onChange);
});
}
function isPeerTransportReady(pc) {
if (!pc) return false;
const conn = pc.connectionState;
const ice = pc.iceConnectionState;
if (conn === 'connected') return true;
if (ice === 'connected' || ice === 'completed') return true;
return false;
}
function waitForPeerConnected(pc, timeoutMs = 10000) {
return new Promise((resolve, reject) => {
if (!pc) {
reject(new Error('Peer connection missing'));
return;
}
if (isPeerTransportReady(pc)) {
resolve();
return;
}
const timer = setTimeout(() => {
cleanup();
reject(new Error('Peer connection timeout'));
}, timeoutMs);
const onState = () => {
if (isPeerTransportReady(pc)) {
cleanup();
resolve();
} else if (
pc.connectionState === 'failed' ||
pc.connectionState === 'closed' ||
pc.iceConnectionState === 'failed'
) {
cleanup();
reject(new Error(`Peer connection ${pc.connectionState || pc.iceConnectionState}`));
}
};
function cleanup() {
clearTimeout(timer);
pc.removeEventListener('connectionstatechange', onState);
pc.removeEventListener('iceconnectionstatechange', onState);
}
pc.addEventListener('connectionstatechange', onState);
pc.addEventListener('iceconnectionstatechange', onState);
});
}
async function configureSenderForLowLatency(sender) {
if (!sender?.getParameters || !sender?.setParameters) return;
const params = sender.getParameters() || {};
const first = (params.encodings && params.encodings[0]) || {};
params.encodings = [
{
...first,
maxBitrate: 64000,
dtx: 'disabled',
},
];
try {
await sender.setParameters(params);
} catch {
// Browser support varies; keep defaults if rejected.
}
}
function waitForOutboundAudioFlow(pc, timeoutMs = 6000) {
return new Promise((resolve, reject) => {
if (!pc) {
reject(new Error('Peer connection missing'));
return;
}
const start = Date.now();
let baseline = -1;
const timer = setInterval(async () => {
if (Date.now() - start > timeoutMs) {
clearInterval(timer);
reject(new Error('WHIP connected but no outbound audio flow'));
return;
}
try {
const senders = pc.getSenders().filter((s) => s.track?.kind === 'audio');
for (const sender of senders) {
const stats = await sender.getStats();
for (const report of stats.values()) {
if (report.type !== 'outbound-rtp' || report.kind !== 'audio') continue;
const sent = Number(report.bytesSent || 0);
const packets = Number(report.packetsSent || 0);
if (baseline < 0) {
baseline = sent;
} else if (sent > baseline + 200 || packets > 5) {
clearInterval(timer);
resolve();
return;
}
}
}
} catch {
// Keep polling until timeout.
}
}, 250);
});
}
function StatusIndicator({ label, active, detail = '' }) {
return (
{label}
{detail || (active ? 'active' : 'idle')}
);
}
function mergeFloatChunks(chunks) {
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const merged = new Float32Array(total);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.length;
}
return merged;
}
function pcm16FromFloat32(samples) {
const out = new Int16Array(samples.length);
for (let i = 0; i < samples.length; i += 1) {
const s = Math.max(-1, Math.min(1, samples[i]));
out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
}
return out;
}
function encodeWavMono16(samples, sampleRate) {
const pcm = pcm16FromFloat32(samples);
const dataSize = pcm.length * 2;
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer);
const writeString = (offset, value) => {
for (let i = 0; i < value.length; i += 1) {
view.setUint8(offset + i, value.charCodeAt(i));
}
};
writeString(0, 'RIFF');
view.setUint32(4, 36 + dataSize, true);
writeString(8, 'WAVE');
writeString(12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, 1, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * 2, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
writeString(36, 'data');
view.setUint32(40, dataSize, true);
let offset = 44;
for (let i = 0; i < pcm.length; i += 1, offset += 2) {
view.setInt16(offset, pcm[i], true);
}
return new Uint8Array(buffer);
}
export default function VipAudioUploadCard({
ownRoverId = '',
audioForwardByRover = {},
playUploadedAudio,
stopUploadedAudio,
startMicWhip,
readyMicWhip,
stopMicWhip,
}) {
const { state: controlState } = useControlSystem();
const { value: vipAudio, save: saveVipAudio } = useSettingsNamespace('vipAudio', {
openMicEnabled: false,
pttMode: 'live',
});
const roverId = String(ownRoverId || '').trim();
const [selectedUpload, setSelectedUpload] = useState(null);
const [working, setWorking] = useState(false);
const openMicEnabled = Boolean(vipAudio?.openMicEnabled);
const pttMode = vipAudio?.pttMode === 'clip' ? 'clip' : 'live';
const clipMode = pttMode === 'clip';
const [micState, setMicState] = useState('idle');
const [clipState, setClipState] = useState('idle');
const [message, setMessage] = useState('');
const streamRef = useRef(null);
const audioTrackRef = useRef(null);
const whipPcRef = useRef(null);
const micActiveRef = useRef(false);
const activeRoverRef = useRef('');
const clipStreamRef = useRef(null);
const clipAudioContextRef = useRef(null);
const clipSourceRef = useRef(null);
const clipProcessorRef = useRef(null);
const clipMuteGainRef = useRef(null);
const clipChunksRef = useRef([]);
const clipRecordingRef = useRef(false);
const clipSampleRateRef = useRef(TARGET_SAMPLE_RATE);
const pttActive = Boolean(controlState?.mic?.pttActive);
const selectedForwardState = useMemo(
() => (roverId ? audioForwardByRover?.[roverId] || null : null),
[audioForwardByRover, roverId],
);
const pipelineConnected = Boolean(
selectedForwardState && selectedForwardState.state !== 'offline' && !selectedForwardState.error,
);
const uploadPlaying = Boolean(
selectedForwardState?.source === 'upload' && selectedForwardState?.state === 'playing',
);
const micRelayActive = Boolean(
selectedForwardState?.source === 'mic-whip' &&
(selectedForwardState?.state === 'starting' || selectedForwardState?.state === 'playing'),
);
const micHot = Boolean(!clipMode && audioTrackRef.current && (openMicEnabled || pttActive));
const whipLinkActive = !clipMode && (micState === 'live' || micState === 'starting');
const clipRecording = clipMode && clipState === 'recording';
const clipSending = clipMode && clipState === 'sending';
const setPttMode = useCallback(
(nextMode) => {
const mode = nextMode === 'clip' ? 'clip' : 'live';
saveVipAudio((current) => ({ ...(current || {}), pttMode: mode }));
},
[saveVipAudio],
);
const handleUploadPlay = async () => {
if (!roverId) {
setMessage('Take control of your rover first.');
return;
}
if (!selectedUpload) {
setMessage('Select an audio file first.');
return;
}
if (selectedUpload.size > MAX_UPLOAD_BYTES) {
setMessage(`File too large (max ${MAX_UPLOAD_BYTES} bytes).`);
return;
}
setWorking(true);
setMessage('');
try {
const buffer = await selectedUpload.arrayBuffer();
const dataBase64 = bytesToBase64(new Uint8Array(buffer));
await playUploadedAudio?.({
roverId,
name: selectedUpload.name,
mime: selectedUpload.type || '',
dataBase64,
});
setMessage('Upload playback started.');
} catch (err) {
setMessage(err?.message || 'Failed to play upload.');
} finally {
setWorking(false);
}
};
const handleUploadStop = async () => {
if (!roverId) {
setMessage('Take control of your rover first.');
return;
}
setWorking(true);
setMessage('');
try {
await stopUploadedAudio?.(roverId);
setMessage('Upload playback stopped.');
} catch (err) {
setMessage(err?.message || 'Failed to stop upload.');
} finally {
setWorking(false);
}
};
const stopMicCapture = useCallback(
async (targetRoverId) => {
const target = String(targetRoverId || activeRoverRef.current || '').trim();
micActiveRef.current = false;
setMicState('idle');
if (whipPcRef.current) {
try {
whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop());
} catch {
// noop
}
try {
whipPcRef.current.close();
} catch {
// noop
}
}
whipPcRef.current = null;
if (streamRef.current) {
try {
streamRef.current.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
}
streamRef.current = null;
audioTrackRef.current = null;
if (target) {
try {
await stopMicWhip?.(target);
} catch {
// noop
}
}
activeRoverRef.current = '';
},
[stopMicWhip],
);
const startWhipMic = useCallback(
async (target) => {
const startPayload = await startMicWhip?.(target);
const whipUrl = String(startPayload?.whipUrl || '').trim();
const token = String(startPayload?.token || '').trim();
if (!whipUrl || !token) {
throw new Error('WHIP endpoint unavailable');
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
},
});
streamRef.current = stream;
const track = stream.getAudioTracks()?.[0];
audioTrackRef.current = track || null;
if (track) {
track.enabled = Boolean(openMicEnabled || pttActive);
}
if (track?.applyConstraints) {
try {
await track.applyConstraints({
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
});
} catch {
// noop
}
}
const pc = new RTCPeerConnection(RTC_CONFIG);
whipPcRef.current = pc;
pc.onconnectionstatechange = () => {
if (!micActiveRef.current) return;
const state = pc.connectionState;
if (state === 'connected') {
setMicState('live');
return;
}
if (state === 'failed' || state === 'disconnected' || state === 'closed') {
setMicState('error');
setMessage(`WHIP transport ${state}.`);
}
};
stream.getAudioTracks().forEach((audioTrack) => {
const sender = pc.addTrack(audioTrack, stream);
configureSenderForLowLatency(sender);
});
const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false });
await pc.setLocalDescription(offer);
await waitForIceGatheringComplete(pc, 1800);
const response = await fetch(whipUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/sdp',
...buildAuthHeader(token),
},
body: pc.localDescription?.sdp || offer.sdp,
});
if (!response.ok) {
throw new Error(`WHIP request failed: ${response.status}`);
}
const answerSdp = await response.text();
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
await waitForPeerConnected(pc, 10000);
await waitForOutboundAudioFlow(pc, 6000);
await readyMicWhip?.(target);
},
[openMicEnabled, pttActive, readyMicWhip, startMicWhip],
);
const teardownClipPipeline = useCallback(async () => {
clipRecordingRef.current = false;
if (clipProcessorRef.current) {
try {
clipProcessorRef.current.disconnect();
} catch {
// noop
}
clipProcessorRef.current.onaudioprocess = null;
}
if (clipSourceRef.current) {
try {
clipSourceRef.current.disconnect();
} catch {
// noop
}
}
if (clipMuteGainRef.current) {
try {
clipMuteGainRef.current.disconnect();
} catch {
// noop
}
}
if (clipStreamRef.current) {
try {
clipStreamRef.current.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
}
if (clipAudioContextRef.current) {
try {
await clipAudioContextRef.current.close();
} catch {
// noop
}
}
clipChunksRef.current = [];
clipProcessorRef.current = null;
clipSourceRef.current = null;
clipMuteGainRef.current = null;
clipStreamRef.current = null;
clipAudioContextRef.current = null;
clipSampleRateRef.current = TARGET_SAMPLE_RATE;
}, []);
const ensureClipPipeline = useCallback(async () => {
if (clipStreamRef.current && clipAudioContextRef.current && clipProcessorRef.current) {
return;
}
setClipState((prev) => (prev === 'recording' ? prev : 'arming'));
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
},
});
const context = new AudioContext({ latencyHint: 'interactive', sampleRate: TARGET_SAMPLE_RATE });
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(1024, 1, 1);
const muteGain = context.createGain();
muteGain.gain.value = 0;
processor.onaudioprocess = (event) => {
if (!clipRecordingRef.current) return;
const input = event.inputBuffer.getChannelData(0);
clipChunksRef.current.push(new Float32Array(input));
};
source.connect(processor);
processor.connect(muteGain);
muteGain.connect(context.destination);
clipStreamRef.current = stream;
clipAudioContextRef.current = context;
clipSourceRef.current = source;
clipProcessorRef.current = processor;
clipMuteGainRef.current = muteGain;
clipSampleRateRef.current = context.sampleRate || TARGET_SAMPLE_RATE;
}, []);
const finishClipRecording = useCallback(
async ({ send }) => {
const wasRecording = clipRecordingRef.current;
clipRecordingRef.current = false;
if (!wasRecording) {
if (!send) setClipState('idle');
return;
}
const chunks = clipChunksRef.current;
clipChunksRef.current = [];
if (!send) {
setClipState('idle');
return;
}
if (!roverId) {
setClipState('idle');
return;
}
if (!chunks.length) {
setClipState('idle');
return;
}
setClipState('sending');
try {
const merged = mergeFloatChunks(chunks);
const wavBytes = encodeWavMono16(merged, clipSampleRateRef.current || TARGET_SAMPLE_RATE);
const dataBase64 = bytesToBase64(wavBytes);
await playUploadedAudio?.({
roverId,
name: `ptt-${Date.now()}.wav`,
mime: 'audio/wav',
dataBase64,
});
setClipState('idle');
} catch (err) {
setClipState('error');
setMessage(err?.message || 'Failed to send PTT clip.');
}
},
[playUploadedAudio, roverId],
);
const startClipRecording = useCallback(async () => {
if (!roverId) {
setMessage('Take control of your rover first.');
return;
}
await ensureClipPipeline();
clipChunksRef.current = [];
clipRecordingRef.current = true;
setClipState('recording');
}, [ensureClipPipeline, roverId]);
useEffect(() => {
let cancelled = false;
async function syncLiveMic() {
if (clipMode) {
await stopMicCapture(activeRoverRef.current || roverId);
return;
}
if (!roverId) {
await stopMicCapture(activeRoverRef.current);
return;
}
if (micActiveRef.current && activeRoverRef.current === roverId) return;
if (!openMicEnabled && !pttActive) return;
try {
setMicState('starting');
setMessage('');
micActiveRef.current = true;
activeRoverRef.current = roverId;
await startWhipMic(roverId);
if (!cancelled) {
setMicState('live');
}
} catch (err) {
if (!cancelled) {
setMicState('error');
setMessage(err?.message || 'Failed to start mic forwarding.');
}
await stopMicCapture(roverId);
}
}
syncLiveMic();
return () => {
cancelled = true;
};
}, [clipMode, openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
useEffect(() => {
const track = audioTrackRef.current;
if (!track) return;
track.enabled = Boolean(!clipMode && (openMicEnabled || pttActive));
}, [clipMode, openMicEnabled, pttActive]);
useEffect(() => {
let cancelled = false;
async function prewarmClip() {
if (!clipMode || !roverId) {
if (clipRecordingRef.current) {
await finishClipRecording({ send: false });
}
setClipState('idle');
return;
}
try {
await ensureClipPipeline();
if (!cancelled && !clipRecordingRef.current) {
setClipState('idle');
}
} catch (err) {
if (!cancelled) {
setClipState('error');
setMessage(err?.message || 'Microphone access failed for clip mode.');
}
}
}
prewarmClip();
return () => {
cancelled = true;
};
}, [clipMode, ensureClipPipeline, finishClipRecording, roverId]);
useEffect(() => {
let cancelled = false;
async function syncClipPtt() {
if (!clipMode) return;
if (!roverId) {
if (clipRecordingRef.current) {
await finishClipRecording({ send: false });
}
return;
}
if (pttActive) {
if (!clipRecordingRef.current) {
try {
setMessage('');
await startClipRecording();
} catch (err) {
if (!cancelled) {
setClipState('error');
setMessage(err?.message || 'Failed to start clip recording.');
}
}
}
} else if (clipRecordingRef.current) {
await finishClipRecording({ send: true });
}
}
syncClipPtt();
return () => {
cancelled = true;
};
}, [clipMode, finishClipRecording, pttActive, roverId, startClipRecording]);
useEffect(
() => () => {
stopMicCapture(activeRoverRef.current || roverId);
teardownClipPipeline();
},
[roverId, stopMicCapture, teardownClipPipeline],
);
return (
PTT Mode
Choose how holding your PTT key behaves: live mic stream, or record-and-send clip.
Audio Upload
{selectedUpload ? (
{selectedUpload.name} ({selectedUpload.size} bytes)
) : (
No file selected
)}
Audio Status
{message ?
{message}
: null}
);
}