This commit is contained in:
legop3
2026-03-21 00:52:20 -04:00
parent 54f8684169
commit aeb4b29b88
9 changed files with 564 additions and 197 deletions
+14 -3
View File
@@ -90,20 +90,31 @@ function MobileFeatureTabs({
}) {
const { session } = useSession();
const { state: controlState } = useControlSystem();
const { value: vipAudio } = useSettingsNamespace('vipAudio', { openMicEnabled: false });
const { value: vipAudio } = useSettingsNamespace('vipAudio', { openMicEnabled: false, pttMode: 'live' });
const vipDotClass = session?.isVerified ? 'bg-emerald-400' : 'bg-amber-400';
const ownRoverId = String(session?.assignment?.roverId || '').trim();
const ownAudioForward = ownRoverId ? session?.audioForward?.[ownRoverId] : null;
const pttActive = Boolean(controlState?.mic?.pttActive);
const openMicEnabled = Boolean(vipAudio?.openMicEnabled);
const pttMode = vipAudio?.pttMode === 'clip' ? 'clip' : 'live';
const vipMicActive = Boolean(
ownRoverId && session?.isVerified && (openMicEnabled || pttActive),
ownRoverId &&
session?.isVerified &&
(pttMode === 'clip' ? pttActive : (openMicEnabled || pttActive)),
);
const vipClipPlaying = Boolean(
ownRoverId &&
session?.isVerified &&
pttMode === 'clip' &&
ownAudioForward?.source === 'upload' &&
ownAudioForward?.state === 'playing',
);
return (
<section className="panel text-base">
<Tabs defaultTab="chat">
<TabList>
<Tab id="chat">Chat</Tab>
<Tab id="vip" highlight={vipMicActive ? 'pink' : 'none'}>
<Tab id="vip" highlight={vipClipPlaying ? 'green' : vipMicActive ? 'pink' : 'none'}>
<span className="inline-flex items-center gap-0.5">
<span>VIP</span>
<span
+14 -3
View File
@@ -113,20 +113,31 @@ function DriveDockPanel() {
export default function RightPaneTabs({ layout, onOpenHelpOverlay }) {
const { session } = useSession();
const { state: controlState } = useControlSystem();
const { value: vipAudio } = useSettingsNamespace('vipAudio', { openMicEnabled: false });
const { value: vipAudio } = useSettingsNamespace('vipAudio', { openMicEnabled: false, pttMode: 'live' });
const vipDotClass = session?.isVerified ? 'bg-emerald-400' : 'bg-red-600';
const ownRoverId = String(session?.assignment?.roverId || '').trim();
const ownAudioForward = ownRoverId ? session?.audioForward?.[ownRoverId] : null;
const pttActive = Boolean(controlState?.mic?.pttActive);
const openMicEnabled = Boolean(vipAudio?.openMicEnabled);
const pttMode = vipAudio?.pttMode === 'clip' ? 'clip' : 'live';
const vipMicActive = Boolean(
ownRoverId && session?.isVerified && (openMicEnabled || pttActive),
ownRoverId &&
session?.isVerified &&
(pttMode === 'clip' ? pttActive : (openMicEnabled || pttActive)),
);
const vipClipPlaying = Boolean(
ownRoverId &&
session?.isVerified &&
pttMode === 'clip' &&
ownAudioForward?.source === 'upload' &&
ownAudioForward?.state === 'playing',
);
return (
<section className="panel text-base">
<Tabs defaultTab="telemetry">
<TabList>
<Tab id="telemetry">Controls</Tab>
<Tab id="vip" highlight={vipMicActive ? 'pink' : 'none'}>
<Tab id="vip" highlight={vipClipPlaying ? 'green' : vipMicActive ? 'pink' : 'none'}>
<span className="inline-flex items-center gap-2">
<span>VIP</span>
<span
+1
View File
@@ -21,6 +21,7 @@ const TAB_VARIANTS = {
const TAB_HIGHLIGHTS = {
none: '',
pink: 'bg-pink-500 text-white border-pink-200 hover:bg-pink-400',
green: 'bg-emerald-500 text-white border-emerald-200 hover:bg-emerald-400',
};
const DEFAULT_VARIANT = 'primary';
+407 -63
View File
@@ -168,6 +168,60 @@ function StatusIndicator({ label, active, detail = '' }) {
);
}
function mergeFloatChunks(chunks) {
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const merged = new Float32Array(total);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.length;
}
return merged;
}
function pcm16FromFloat32(samples) {
const out = new Int16Array(samples.length);
for (let i = 0; i < samples.length; i += 1) {
const s = Math.max(-1, Math.min(1, samples[i]));
out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
}
return out;
}
function encodeWavMono16(samples, sampleRate) {
const pcm = pcm16FromFloat32(samples);
const dataSize = pcm.length * 2;
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer);
const writeString = (offset, value) => {
for (let i = 0; i < value.length; i += 1) {
view.setUint8(offset + i, value.charCodeAt(i));
}
};
writeString(0, 'RIFF');
view.setUint32(4, 36 + dataSize, true);
writeString(8, 'WAVE');
writeString(12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, 1, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * 2, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
writeString(36, 'data');
view.setUint32(40, dataSize, true);
let offset = 44;
for (let i = 0; i < pcm.length; i += 1, offset += 2) {
view.setInt16(offset, pcm[i], true);
}
return new Uint8Array(buffer);
}
export default function VipAudioUploadCard({
ownRoverId = '',
audioForwardByRover = {},
@@ -178,19 +232,37 @@ export default function VipAudioUploadCard({
stopMicWhip,
}) {
const { state: controlState } = useControlSystem();
const { value: vipAudio, save: saveVipAudio } = useSettingsNamespace('vipAudio', { openMicEnabled: false });
const { value: vipAudio, save: saveVipAudio } = useSettingsNamespace('vipAudio', {
openMicEnabled: false,
pttMode: 'live',
});
const roverId = String(ownRoverId || '').trim();
const [selectedUpload, setSelectedUpload] = useState(null);
const [working, setWorking] = useState(false);
const openMicEnabled = Boolean(vipAudio?.openMicEnabled);
const pttMode = vipAudio?.pttMode === 'clip' ? 'clip' : 'live';
const clipMode = pttMode === 'clip';
const [micState, setMicState] = useState('idle');
const [clipState, setClipState] = useState('idle');
const [message, setMessage] = useState('');
const streamRef = useRef(null);
const audioTrackRef = useRef(null);
const whipPcRef = useRef(null);
const micActiveRef = useRef(false);
const activeRoverRef = useRef('');
const clipStreamRef = useRef(null);
const clipAudioContextRef = useRef(null);
const clipSourceRef = useRef(null);
const clipProcessorRef = useRef(null);
const clipMuteGainRef = useRef(null);
const clipChunksRef = useRef([]);
const clipRecordingRef = useRef(false);
const clipSampleRateRef = useRef(TARGET_SAMPLE_RATE);
const pttActive = Boolean(controlState?.mic?.pttActive);
const selectedForwardState = useMemo(
@@ -208,8 +280,18 @@ export default function VipAudioUploadCard({
selectedForwardState?.source === 'mic-whip' &&
(selectedForwardState?.state === 'starting' || selectedForwardState?.state === 'playing'),
);
const micHot = Boolean(audioTrackRef.current && (openMicEnabled || pttActive));
const whipLinkActive = micState === 'live' || micState === 'starting';
const micHot = Boolean(!clipMode && audioTrackRef.current && (openMicEnabled || pttActive));
const whipLinkActive = !clipMode && (micState === 'live' || micState === 'starting');
const clipRecording = clipMode && clipState === 'recording';
const clipSending = clipMode && clipState === 'sending';
const setPttMode = useCallback(
(nextMode) => {
const mode = nextMode === 'clip' ? 'clip' : 'live';
saveVipAudio((current) => ({ ...(current || {}), pttMode: mode }));
},
[saveVipAudio],
);
const handleUploadPlay = async () => {
if (!roverId) {
@@ -383,10 +465,162 @@ export default function VipAudioUploadCard({
[openMicEnabled, pttActive, readyMicWhip, startMicWhip],
);
const teardownClipPipeline = useCallback(async () => {
clipRecordingRef.current = false;
if (clipProcessorRef.current) {
try {
clipProcessorRef.current.disconnect();
} catch {
// noop
}
clipProcessorRef.current.onaudioprocess = null;
}
if (clipSourceRef.current) {
try {
clipSourceRef.current.disconnect();
} catch {
// noop
}
}
if (clipMuteGainRef.current) {
try {
clipMuteGainRef.current.disconnect();
} catch {
// noop
}
}
if (clipStreamRef.current) {
try {
clipStreamRef.current.getTracks().forEach((track) => track.stop());
} catch {
// noop
}
}
if (clipAudioContextRef.current) {
try {
await clipAudioContextRef.current.close();
} catch {
// noop
}
}
clipChunksRef.current = [];
clipProcessorRef.current = null;
clipSourceRef.current = null;
clipMuteGainRef.current = null;
clipStreamRef.current = null;
clipAudioContextRef.current = null;
clipSampleRateRef.current = TARGET_SAMPLE_RATE;
}, []);
const ensureClipPipeline = useCallback(async () => {
if (clipStreamRef.current && clipAudioContextRef.current && clipProcessorRef.current) {
return;
}
setClipState((prev) => (prev === 'recording' ? prev : 'arming'));
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
sampleRate: TARGET_SAMPLE_RATE,
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
},
});
const context = new AudioContext({ latencyHint: 'interactive', sampleRate: TARGET_SAMPLE_RATE });
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(1024, 1, 1);
const muteGain = context.createGain();
muteGain.gain.value = 0;
processor.onaudioprocess = (event) => {
if (!clipRecordingRef.current) return;
const input = event.inputBuffer.getChannelData(0);
clipChunksRef.current.push(new Float32Array(input));
};
source.connect(processor);
processor.connect(muteGain);
muteGain.connect(context.destination);
clipStreamRef.current = stream;
clipAudioContextRef.current = context;
clipSourceRef.current = source;
clipProcessorRef.current = processor;
clipMuteGainRef.current = muteGain;
clipSampleRateRef.current = context.sampleRate || TARGET_SAMPLE_RATE;
}, []);
const finishClipRecording = useCallback(
async ({ send }) => {
const wasRecording = clipRecordingRef.current;
clipRecordingRef.current = false;
if (!wasRecording) {
if (!send) setClipState('idle');
return;
}
const chunks = clipChunksRef.current;
clipChunksRef.current = [];
if (!send) {
setClipState('idle');
return;
}
if (!roverId) {
setClipState('idle');
return;
}
if (!chunks.length) {
setClipState('idle');
return;
}
setClipState('sending');
try {
const merged = mergeFloatChunks(chunks);
const wavBytes = encodeWavMono16(merged, clipSampleRateRef.current || TARGET_SAMPLE_RATE);
const dataBase64 = bytesToBase64(wavBytes);
await playUploadedAudio?.({
roverId,
name: `ptt-${Date.now()}.wav`,
mime: 'audio/wav',
dataBase64,
});
setClipState('idle');
} catch (err) {
setClipState('error');
setMessage(err?.message || 'Failed to send PTT clip.');
}
},
[playUploadedAudio, roverId],
);
const startClipRecording = useCallback(async () => {
if (!roverId) {
setMessage('Take control of your rover first.');
return;
}
await ensureClipPipeline();
clipChunksRef.current = [];
clipRecordingRef.current = true;
setClipState('recording');
}, [ensureClipPipeline, roverId]);
useEffect(() => {
let cancelled = false;
async function syncMicState() {
async function syncLiveMic() {
if (clipMode) {
await stopMicCapture(activeRoverRef.current || roverId);
return;
}
if (!roverId) {
await stopMicCapture(activeRoverRef.current);
return;
@@ -412,92 +646,202 @@ export default function VipAudioUploadCard({
}
}
syncMicState();
syncLiveMic();
return () => {
cancelled = true;
};
}, [openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
}, [clipMode, openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
useEffect(() => {
const track = audioTrackRef.current;
if (!track) return;
track.enabled = Boolean(openMicEnabled || pttActive);
}, [openMicEnabled, pttActive]);
track.enabled = Boolean(!clipMode && (openMicEnabled || pttActive));
}, [clipMode, openMicEnabled, pttActive]);
useEffect(() => {
let cancelled = false;
async function prewarmClip() {
if (!clipMode || !roverId) {
if (clipRecordingRef.current) {
await finishClipRecording({ send: false });
}
setClipState('idle');
return;
}
try {
await ensureClipPipeline();
if (!cancelled && !clipRecordingRef.current) {
setClipState('idle');
}
} catch (err) {
if (!cancelled) {
setClipState('error');
setMessage(err?.message || 'Microphone access failed for clip mode.');
}
}
}
prewarmClip();
return () => {
cancelled = true;
};
}, [clipMode, ensureClipPipeline, finishClipRecording, roverId]);
useEffect(() => {
let cancelled = false;
async function syncClipPtt() {
if (!clipMode) return;
if (!roverId) {
if (clipRecordingRef.current) {
await finishClipRecording({ send: false });
}
return;
}
if (pttActive) {
if (!clipRecordingRef.current) {
try {
setMessage('');
await startClipRecording();
} catch (err) {
if (!cancelled) {
setClipState('error');
setMessage(err?.message || 'Failed to start clip recording.');
}
}
}
} else if (clipRecordingRef.current) {
await finishClipRecording({ send: true });
}
}
syncClipPtt();
return () => {
cancelled = true;
};
}, [clipMode, finishClipRecording, pttActive, roverId, startClipRecording]);
useEffect(
() => () => {
stopMicCapture(activeRoverRef.current || roverId);
teardownClipPipeline();
},
[roverId, stopMicCapture],
[roverId, stopMicCapture, teardownClipPipeline],
);
return (
<div className="grid gap-0.5 grid-cols-1 lg:grid-cols-2">
<section className="surface h-full">
<div className="grid h-full gap-0.5 grid-rows-[auto_auto_1fr_auto]">
<p className="text-sm text-slate-200 text-center">Audio Upload</p>
<label className="mx-auto grid w-full max-w-sm gap-0.5 text-xs text-slate-300 text-center">
<span>Audio file (mp3 / wav / ogg)</span>
<input
className={`${fieldClass} text-center`}
type="file"
accept=".mp3,.wav,.ogg,audio/mpeg,audio/wav,audio/ogg"
disabled={working || !roverId}
onChange={(event) => setSelectedUpload(event.target.files?.[0] || null)}
/>
</label>
<div className="mx-auto w-full max-w-sm text-center">
{selectedUpload ? (
<div className="surface-muted text-xs text-slate-300">
{selectedUpload.name} ({selectedUpload.size} bytes)
</div>
) : (
<div className="surface-muted text-xs text-slate-500">No file selected</div>
)}
<div className="grid gap-0.5">
<div className="grid gap-0.5 grid-cols-1 lg:grid-cols-2">
<section className="surface h-full">
<div className="grid h-full gap-0.5 grid-rows-[auto_auto_1fr]">
<p className="text-sm text-slate-200 text-center">PTT Mode</p>
<p className="text-xs text-slate-400 text-center">
Choose how holding your PTT key behaves: live mic stream, or record-and-send clip.
</p>
<div className="grid grid-cols-2 gap-0.5">
<button
type="button"
className={`button-dark w-full text-sm ${!clipMode ? 'bg-emerald-500 text-white hover:bg-emerald-500' : ''}`}
onClick={() => setPttMode('live')}
>
Live Mic (WHIP)
</button>
<button
type="button"
className={`button-dark w-full text-sm ${clipMode ? 'bg-emerald-500 text-white hover:bg-emerald-500' : ''}`}
onClick={() => setPttMode('clip')}
>
Record Clip
</button>
</div>
</div>
</section>
<div className="flex justify-center gap-0.5">
<button type="button" className="button-dark text-sm" disabled={working || !roverId} onClick={handleUploadPlay}>
{working ? 'Working...' : 'Play Upload'}
</button>
<button type="button" className="button-dark text-sm" disabled={working || !roverId} onClick={handleUploadStop}>
Stop
</button>
<section className="surface h-full">
<div className="grid h-full gap-0.5 grid-rows-[auto_1fr]">
<p className="text-sm text-slate-200 text-center">PTT Clip Status</p>
<div className="grid gap-0.5 content-start">
<StatusIndicator label="Mode" active={clipMode} detail={clipMode ? 'record clip' : 'live mic'} />
<StatusIndicator label="PTT hold" active={pttActive} detail={pttActive ? 'held' : 'released'} />
<StatusIndicator label="Recording" active={clipRecording} detail={clipRecording ? 'capturing' : 'idle'} />
<StatusIndicator label="Playback" active={clipMode && uploadPlaying} detail={clipMode && uploadPlaying ? 'playing' : 'idle'} />
</div>
</div>
</div>
</section>
</section>
</div>
<section className="surface h-full">
<div className="grid h-full gap-0.5 grid-rows-[auto_auto_1fr]">
<p className="text-sm text-slate-200 text-center">Live Microphone</p>
<label className="surface-muted mx-auto flex w-full max-w-sm items-center justify-center gap-0.5 px-0.5 py-0.5 text-xs text-slate-300">
<input
type="checkbox"
checked={openMicEnabled}
disabled={!roverId}
onChange={(event) => saveVipAudio((current) => ({ ...(current || {}), openMicEnabled: Boolean(event.target.checked) }))}
/>
<span>Open mic</span>
</label>
<div className="grid gap-0.5 content-start">
<StatusIndicator label="WHIP link" active={whipLinkActive} detail={micState} />
<StatusIndicator label="Mic hot" active={micHot} detail={micHot ? 'transmitting' : 'muted'} />
<div className="surface-muted text-center text-xs text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'm'} (hold)</div>
<div className="grid gap-0.5 grid-cols-1 lg:grid-cols-2">
<section className="surface h-full">
<div className="grid h-full gap-0.5 grid-rows-[auto_auto_1fr_auto]">
<p className="text-sm text-slate-200 text-center">Audio Upload</p>
<label className="mx-auto grid w-full max-w-sm gap-0.5 text-xs text-slate-300 text-center">
<span>Audio file (mp3 / wav / ogg)</span>
<input
className={`${fieldClass} text-center`}
type="file"
accept=".mp3,.wav,.ogg,audio/mpeg,audio/wav,audio/ogg"
disabled={working || !roverId}
onChange={(event) => setSelectedUpload(event.target.files?.[0] || null)}
/>
</label>
<div className="mx-auto w-full max-w-sm text-center">
{selectedUpload ? (
<div className="surface-muted text-xs text-slate-300">
{selectedUpload.name} ({selectedUpload.size} bytes)
</div>
) : (
<div className="surface-muted text-xs text-slate-500">No file selected</div>
)}
</div>
<div className="flex justify-center gap-0.5">
<button type="button" className="button-dark text-sm" disabled={working || !roverId} onClick={handleUploadPlay}>
{working ? 'Working...' : 'Play Upload'}
</button>
<button type="button" className="button-dark text-sm" disabled={working || !roverId} onClick={handleUploadStop}>
Stop
</button>
</div>
</div>
</div>
</section>
</section>
<section className="surface lg:col-span-2">
<section className="surface h-full">
<div className="grid h-full gap-0.5 grid-rows-[auto_auto_1fr]">
<p className="text-sm text-slate-200 text-center">Live Microphone</p>
<label className="surface-muted mx-auto flex w-full max-w-sm items-center justify-center gap-0.5 px-0.5 py-0.5 text-xs text-slate-300">
<input
type="checkbox"
checked={openMicEnabled}
disabled={!roverId || clipMode}
onChange={(event) =>
saveVipAudio((current) => ({ ...(current || {}), openMicEnabled: Boolean(event.target.checked) }))
}
/>
<span>{clipMode ? 'Open mic (live mode only)' : 'Open mic'}</span>
</label>
<div className="grid gap-0.5 content-start">
<StatusIndicator label="WHIP link" active={whipLinkActive} detail={micState} />
<StatusIndicator label="Mic hot" active={micHot} detail={micHot ? 'transmitting' : 'muted'} />
<div className="surface-muted text-center text-xs text-slate-400">PTT key: {controlState?.keymap?.micPtt?.[0] || 'm'} (hold)</div>
</div>
</div>
</section>
</div>
<section className="surface">
<div className="space-y-0.5">
<p className="text-sm text-slate-200 text-center">Audio Status</p>
<div className="grid gap-0.5 grid-cols-1 sm:grid-cols-2 lg:grid-cols-5">
<div className="grid gap-0.5 grid-cols-1 sm:grid-cols-2 lg:grid-cols-6">
<StatusIndicator label="Forward pipe" active={pipelineConnected} detail={pipelineConnected ? 'connected' : 'offline'} />
<StatusIndicator label="Upload playback" active={uploadPlaying} detail={uploadPlaying ? 'playing' : 'idle'} />
<StatusIndicator label="Mic relay" active={micRelayActive} detail={micRelayActive ? 'active' : 'idle'} />
<StatusIndicator label="WHIP transport" active={whipLinkActive} detail={micState} />
<StatusIndicator label="Mic transmit" active={micHot} detail={micHot ? 'hot' : 'muted'} />
<StatusIndicator label="Clip recording" active={clipRecording} detail={clipRecording ? 'recording' : 'idle'} />
<StatusIndicator label="Clip sending" active={clipSending} detail={clipSending ? 'sending' : 'idle'} />
</div>
{message ? <div className="text-xs text-slate-400 text-center">{message}</div> : null}
</div>