Stratawiz
/
Eventbuzz_Concierge
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert unintended avatar asset changes

This commit is contained in:
Tanuja Pansare 2026-10-19 14:21:33 +05:30
parent 6c80a74ee5
commit cb3d977d10
6 changed files with 204 additions and 1209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
public/embed-avatar.html
View File

@ -9,11 +9,10 @@
<body> <body>
<div id="eba-root"> <div id="eba-root">
<section class="eba-avatar" aria-label="Concierge avatar"> <section class="eba-avatar" aria-label="Concierge avatar">
<div id="eba-avatar-stage"></div> <video id="eba-video" autoplay playsinline muted></video>
<audio id="eba-audio" autoplay playsinline></audio>
<audio id="eba-audio" autoplay playsinline></audio> <canvas id="eba-wave" aria-hidden="true"></canvas>
<div id="eba-avatar-status">Starting…</div>
<div id="eba-avatar-status">Starting…</div>
</section> </section>
<section class="eba-chat" aria-label="Concierge chat"> <section class="eba-chat" aria-label="Concierge chat">
<header class="eba-chat-header"> <header class="eba-chat-header">
@ -43,14 +42,6 @@
}; };
</script> </script>
<script src="https://cdn.jsdelivr.net/npm/livekit-client@2.15.14/dist/livekit-client.umd.js"></script> <script src="https://cdn.jsdelivr.net/npm/livekit-client@2.15.14/dist/livekit-client.umd.js"></script>
<script src="/embed/avatar/avatar3d.js"></script> <script type="module" src="/embed/avatar-page.js?v=9"></script>
<script type="importmap">
{
"imports": {
"three": "https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js"
}
}
</script>
<script type="module" src="/embed/avatar-page.js?v=11"></script>
</body> </body>
</html> </html>

33
public/embed/avatar-page.css
View File

@ -29,32 +29,17 @@ body {
} }
.eba-avatar { .eba-avatar {
position: relative; position: relative;
background: #020617; background: #020617;
min-width: 0; min-width: 0;
min-height: 0; min-height: 0;
overflow: hidden;
} }
#eba-avatar-stage { #eba-video {
position: absolute; width: 100%;
inset: 0; height: 100%;
width: 100%; object-fit: cover;
height: 100%; background: #020617;
}
#eba-avatar-stage canvas {
display: block;
width: 100%;
height: 100%;
}
#eba-audio {
position: absolute;
width: 1px;
height: 1px;
opacity: 0;
pointer-events: none;
} }
#eba-audio { #eba-audio {

617
public/embed/avatar-page.js
View File

@ -4,10 +4,8 @@ const apiBase = String(cfg.apiBase || '').replace(/\/$/, '');
const Livekit = window.LivekitClient || window.livekit; const Livekit = window.LivekitClient || window.livekit;
const videoEl = document.getElementById('eba-video'); const videoEl = document.getElementById('eba-video');
// const audioEl = document.getElementById('eba-audio');
// const waveEl = document.getElementById('eba-wave');
const audioEl = document.getElementById('eba-audio'); const audioEl = document.getElementById('eba-audio');
const avatarStageEl = document.getElementById('eba-avatar-stage'); const waveEl = document.getElementById('eba-wave');
const messagesEl = document.getElementById('eba-messages'); const messagesEl = document.getElementById('eba-messages');
const errorEl = document.getElementById('eba-error'); const errorEl = document.getElementById('eba-error');
const formEl = document.getElementById('eba-form'); const formEl = document.getElementById('eba-form');
@ -22,10 +20,10 @@ const subEl = document.getElementById('eba-sub');
let sessionId = null; let sessionId = null;
let busy = false; let busy = false;
let livekitRoom = null; let livekitRoom = null;
// let simliClient = null; let simliClient = null;
let started = false; let started = false;
let chatReady = false; let chatReady = false;
// let pipedTrackId = null; let pipedTrackId = null;
let seenTranscriptIds = {}; let seenTranscriptIds = {};
let chipsEl = null; let chipsEl = null;
const mutedLivekitAudioEls = []; const mutedLivekitAudioEls = [];
@ -34,24 +32,22 @@ let pendingQuickReplies = [];
let greetingShown = false; let greetingShown = false;
let avatarReadySent = false; let avatarReadySent = false;
let lastAgentMediaTrack = null; let lastAgentMediaTrack = null;
// let lastAgentTrackClone = null; let lastAgentTrackClone = null;
let sessionAlive = false; let sessionAlive = false;
// let simliRestarting = false; let simliRestarting = false;
// let simliIdle = false; let simliIdle = false;
// let simliGeneration = 0; let simliGeneration = 0;
let videoAvatarOn = false; let videoAvatarOn = false;
let waveAudioCtx = null; let waveAudioCtx = null;
let userAnalyser = null; let userAnalyser = null;
let agentAnalyser = null; let agentAnalyser = null;
let avatarAudioSource = null;
let avatarAudioAnalyser = null;
let waveRaf = 0; let waveRaf = 0;
// let simliInputEnabled = false; let simliInputEnabled = false;
let agentPipelineState = ''; let agentPipelineState = '';
let micListening = true; let micListening = true;
let micPublished = false; let micPublished = false;
let micResumeTimer = 0; let micResumeTimer = 0;
// const SIMLI_AUDIO_BUFFER = 3000; const SIMLI_AUDIO_BUFFER = 3000;
const AGENT_STATE_ATTR = 'lk.agent.state'; const AGENT_STATE_ATTR = 'lk.agent.state';
function setStatus(text) { function setStatus(text) {
@ -263,10 +259,6 @@ async function signalAvatarReady(room) {
try { try {
if (typeof room.localParticipant.publishData === 'function') { if (typeof room.localParticipant.publishData === 'function') {
await room.localParticipant.publishData(payload, { reliable: true }); await room.localParticipant.publishData(payload, { reliable: true });
console.log('[DEBUG] avatar-ready SENT', {
identity: room.localParticipant.identity,
room: room.name,
});
} }
} catch (err) { } catch (err) {
console.warn('[avatar] ready signal failed', err); console.warn('[avatar] ready signal failed', err);
@ -314,7 +306,7 @@ async function sendMessage() {
inputEl.value = ''; inputEl.value = '';
renderChips([]); renderChips([]);
addMessage('user', text); addMessage('user', text);
// ensureSimliConnected('chat'); ensureSimliConnected('chat');
setBusy(true); setBusy(true);
try { try {
const data = await api(`/public/concierge/${encodeURIComponent(botId)}/chat`, { const data = await api(`/public/concierge/${encodeURIComponent(botId)}/chat`, {
@ -453,28 +445,28 @@ async function setMicListening(on) {
function applyAgentState(state) { function applyAgentState(state) {
const next = String(state || '').toLowerCase(); const next = String(state || '').toLowerCase();
if (!next) return; if (!next) return;
agentPipelineState = next; agentPipelineState = next;
const speaking = next === 'speaking'; const speaking = next === 'speaking';
const holdMic = speaking || next === 'thinking'; const holdMic = speaking || next === 'thinking';
if (videoAvatarOn) {
if (speaking && lastAgentMediaTrack) {
if (!simliInputEnabled) connectSimliTts(lastAgentMediaTrack);
} else if (!speaking) {
disconnectSimliTts();
}
}
if (holdMic) { if (holdMic) {
setMicListening(false); setMicListening(false);
} else { } else {
micResumeTimer = window.setTimeout( micResumeTimer = window.setTimeout(() => setMicListening(true), 700);
() => setMicListening(true),
700
);
} }
if (next === 'thinking') { if (simliIdle || simliRestarting) return;
setStatus('Thinking'); if (next === 'thinking') setStatus('Thinking');
} else if (speaking) { else if (speaking) setStatus('Speaking');
setStatus('Speaking'); else setStatus('Listening');
} else {
setStatus('Listening');
}
} }
function makeSimliClient(SimliClient, LogLevel, token) { function makeSimliClient(SimliClient, LogLevel, token) {
@ -552,106 +544,24 @@ function ensureWaveAudioCtx() {
if (waveAudioCtx.state === 'suspended') waveAudioCtx.resume().catch(() => {}); if (waveAudioCtx.state === 'suspended') waveAudioCtx.resume().catch(() => {});
return waveAudioCtx; return waveAudioCtx;
} }
function hookAvatarAnalyser(mediaTrack) {
if (!mediaTrack) {
return;
}
const ctx = ensureWaveAudioCtx();
if (!ctx) {
console.warn(
'[avatar3d] AudioContext unavailable'
);
return;
}
try {
if (avatarAudioSource) {
try {
avatarAudioSource.disconnect();
} catch {
// ignore
}
avatarAudioSource = null;
}
const clone =
typeof mediaTrack.clone === 'function'
? mediaTrack.clone()
: mediaTrack;
const stream =
new MediaStream([clone]);
avatarAudioSource =
ctx.createMediaStreamSource(
stream
);
avatarAudioAnalyser =
ctx.createAnalyser();
avatarAudioAnalyser.fftSize = 256;
avatarAudioAnalyser.smoothingTimeConstant = 0.72;
avatarAudioSource.connect(
avatarAudioAnalyser
);
if (
window.EBAvatar3D &&
typeof window.EBAvatar3D.setAudioAnalyser ===
'function'
) {
window.EBAvatar3D.setAudioAnalyser(
avatarAudioAnalyser
);
}
console.info(
'[avatar3d] agent audio connected to avatar'
);
} catch (err) {
console.warn(
'[avatar3d] audio analyser failed',
err
);
}
}
function hookWaveAnalyser(mediaTrack, slot) { function hookWaveAnalyser(mediaTrack, slot) {
if (!mediaTrack || videoAvatarOn) return; if (!mediaTrack || videoAvatarOn) return;
const ctx = ensureWaveAudioCtx(); const ctx = ensureWaveAudioCtx();
if (!ctx) return; if (!ctx) return;
try { try {
const clone = const clone = typeof mediaTrack.clone === 'function' ? mediaTrack.clone() : mediaTrack;
typeof mediaTrack.clone === 'function' const source = ctx.createMediaStreamSource(new MediaStream([clone]));
? mediaTrack.clone()
: mediaTrack;
const source = ctx.createMediaStreamSource(
new MediaStream([clone])
);
const analyser = ctx.createAnalyser(); const analyser = ctx.createAnalyser();
analyser.fftSize = 256; analyser.fftSize = 256;
analyser.smoothingTimeConstant = 0.72; analyser.smoothingTimeConstant = 0.72;
source.connect(analyser); source.connect(analyser);
if (slot === 'user') userAnalyser = analyser;
if (slot === 'user') { else agentAnalyser = analyser;
userAnalyser = analyser;
} else {
agentAnalyser = analyser;
}
} catch (err) { } catch (err) {
console.warn('[avatar] wave analyser failed', err); console.warn('[avatar] wave analyser failed', err);
} }
startWaveLoop();
} }
function analyserSnapshot(analyser) { function analyserSnapshot(analyser) {
@ -746,181 +656,150 @@ function applyStageMode() {
} }
function handleIncomingAudio(track, participant, room) { function handleIncomingAudio(track, participant, room) {
console.log('[DEBUG] handleIncomingAudio()', { if (!track || String(track.kind || '').toLowerCase() !== 'audio') return;
kind: track?.kind, if (isLocalParticipant(room, participant)) return;
participant: participant?.identity, attachRemoteAudio(track, true);
isLocal: isLocalParticipant(room, participant) const media = mediaTrackFrom(track);
}); lastAgentMediaTrack = media;
if ( if (videoAvatarOn) {
!track || if (agentPipelineState === 'speaking') connectSimliTts(media);
String(track.kind || '').toLowerCase() !== 'audio' } else {
) { hookWaveAnalyser(media, 'agent');
return; setTimeout(() => signalAvatarReady(room), 400);
} }
if (isLocalParticipant(room, participant)) {
return;
}
// --------------------------------------------
// Authoritative audio playback
// --------------------------------------------
console.log('[DEBUG] Attaching agent audio');
attachRemoteAudio(track, true);
// --------------------------------------------
// Same audio -> 3D avatar
// --------------------------------------------
const media = mediaTrackFrom(track);
console.log('[DEBUG] handleIncomingAudio()', {
kind: track?.kind,
participant: participant?.identity,
isLocal: isLocalParticipant(room, participant)
});
lastAgentMediaTrack = media;
if (media) {
hookAvatarAnalyser(media);
}
setTimeout(() => {
signalAvatarReady(room);
}, 400);
} }
// function ensureSimliConnected(reason) { function ensureSimliConnected(reason) {
// if (!videoAvatarOn) return; if (!videoAvatarOn) return;
// if (!sessionAlive || !livekitRoom || simliRestarting) return; if (!sessionAlive || !livekitRoom || simliRestarting) return;
// if (!simliIdle && simliClient) return; if (!simliIdle && simliClient) return;
// restartSimliAvatar(reason || 'session-restart'); restartSimliAvatar(reason || 'session-restart');
// } }
// async function restartSimliAvatar(reason) { async function restartSimliAvatar(reason) {
// if (!sessionAlive || simliRestarting || !livekitRoom) return; if (!sessionAlive || simliRestarting || !livekitRoom) return;
// simliRestarting = true; simliRestarting = true;
// // simliGeneration += 1; simliGeneration += 1;
// setStatus('Reconnecting avatar…'); setStatus('Reconnecting avatar…');
// console.warn('[avatar] restarting simli', reason); console.warn('[avatar] restarting simli', reason);
// try { try {
// const prev = simliClient; const prev = simliClient;
// simliClient = null; simliClient = null;
// if (prev && typeof prev.stop === 'function') { if (prev && typeof prev.stop === 'function') {
// await prev.stop().catch(() => {}); await prev.stop().catch(() => {});
// } }
// const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, { const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, {
// method: 'POST', method: 'POST',
// body: {}, body: {},
// }); });
// if (!simli?.sessionToken) throw new Error('Avatar session token was empty'); if (!simli?.sessionToken) throw new Error('Avatar session token was empty');
// const { SimliClient, LogLevel } = await loadSimliClient(); const { SimliClient, LogLevel } = await loadSimliClient();
// const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken); const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken);
// bindSimliEvents(next); bindSimliEvents(next);
// simliClient = next; simliClient = next;
// await next.start(); await next.start();
// const played = await resumeSimliAudio(); const played = await resumeSimliAudio();
// pipedTrackId = null; pipedTrackId = null;
// if (played) muteLivekitPlayback(); if (played) muteLivekitPlayback();
// else unmuteLivekitFallback(); else unmuteLivekitFallback();
// // simliIdle = false; simliIdle = false;
// applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState); applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
// } catch (err) { } catch (err) {
// console.warn('[avatar] simli restart failed', err); console.warn('[avatar] simli restart failed', err);
// simliIdle = true; simliIdle = true;
// setStatus('Avatar disconnected'); setStatus('Avatar disconnected');
// } finally { } finally {
// simliRestarting = false; simliRestarting = false;
// } }
// } }
// function bindSimliEvents(client) { function bindSimliEvents(client) {
// if (!client || typeof client.on !== 'function') return; if (!client || typeof client.on !== 'function') return;
// const gen = simliGeneration; const gen = simliGeneration;
// let ready = false; let ready = false;
// client.on('start', () => { client.on('start', () => {
// if (gen !== simliGeneration) return; if (gen !== simliGeneration) return;
// ready = true; ready = true;
// // simliIdle = false; simliIdle = false;
// muteLivekitPlayback(); muteLivekitPlayback();
// unmuteSimliPlayback(); unmuteSimliPlayback();
// if (audioEl?.play) audioEl.play().catch(() => {}); if (audioEl?.play) audioEl.play().catch(() => {});
// applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState); applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
// }); });
// client.on('stop', () => { client.on('stop', () => {
// if (gen !== simliGeneration) return; if (gen !== simliGeneration) return;
// simliIdle = true; simliIdle = true;
// setStatus('Idle — speak to continue'); setStatus('Idle — speak to continue');
// }); });
// client.on('error', (message) => { client.on('error', (message) => {
// if (gen !== simliGeneration) return; if (gen !== simliGeneration) return;
// console.warn('[avatar] simli error', message); console.warn('[avatar] simli error', message);
// if (sessionAlive && ready) restartSimliAvatar(message || 'error'); if (sessionAlive && ready) restartSimliAvatar(message || 'error');
// }); });
// } }
// async function loadSimliClient() { async function loadSimliClient() {
// const mod = await import('/embed/simli-client.js'); const mod = await import('/embed/simli-client.js');
// const SimliClient = mod.SimliClient || mod.default?.SimliClient || mod.default; const SimliClient = mod.SimliClient || mod.default?.SimliClient || mod.default;
// if (typeof SimliClient !== 'function') { if (typeof SimliClient !== 'function') {
// throw new Error('SIMLI client failed to load'); throw new Error('SIMLI client failed to load');
// } }
// return { SimliClient, LogLevel: mod.LogLevel || mod.default?.LogLevel }; return { SimliClient, LogLevel: mod.LogLevel || mod.default?.LogLevel };
// } }
// async function resumeSimliAudio() { async function resumeSimliAudio() {
// const ctx = simliClient?.audioContext; const ctx = simliClient?.audioContext;
// if (ctx && ctx.state === 'suspended' && typeof ctx.resume === 'function') { if (ctx && ctx.state === 'suspended' && typeof ctx.resume === 'function') {
// await ctx.resume().catch(() => {}); await ctx.resume().catch(() => {});
// } }
// unmuteSimliPlayback(); unmuteSimliPlayback();
// let played = false; let played = false;
// if (audioEl?.play) { if (audioEl?.play) {
// played = await audioEl.play().then(() => true).catch(() => false); played = await audioEl.play().then(() => true).catch(() => false);
// } }
// if (videoEl?.play) videoEl.play().catch(() => {}); if (videoEl?.play) videoEl.play().catch(() => {});
// return played; return played;
// } }
// async function startSimliOverlay() { async function startSimliOverlay() {
// if (!videoAvatarOn || simliRestarting) return; if (!videoAvatarOn || simliRestarting) return;
// simliRestarting = true; simliRestarting = true;
// try { try {
// setStatus('Connecting avatar…'); setStatus('Connecting avatar…');
// const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, { const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, {
// method: 'POST', method: 'POST',
// body: {}, body: {},
// }); });
// if (!simli?.sessionToken) throw new Error('Avatar session token was empty'); if (!simli?.sessionToken) throw new Error('Avatar session token was empty');
// const { SimliClient, LogLevel } = await loadSimliClient(); const { SimliClient, LogLevel } = await loadSimliClient();
// const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken); const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken);
// bindSimliEvents(next); bindSimliEvents(next);
// simliClient = next; simliClient = next;
// await next.start(); await next.start();
// const played = await resumeSimliAudio(); const played = await resumeSimliAudio();
// pipedTrackId = null; pipedTrackId = null;
// if (played) muteLivekitPlayback(); if (played) muteLivekitPlayback();
// else unmuteLivekitFallback(); else unmuteLivekitFallback();
// applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState); applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
// } catch (err) { } catch (err) {
// console.warn('[avatar] simli overlay failed, voice continues', err); console.warn('[avatar] simli overlay failed, voice continues', err);
// simliClient = null; simliClient = null;
// unmuteLivekitFallback(); unmuteLivekitFallback();
// setStatus('Voice only'); setStatus('Voice only');
// } finally { } finally {
// simliRestarting = false; simliRestarting = false;
// } }
// } }
async function teardownMedia() { async function teardownMedia() {
sessionAlive = false; sessionAlive = false;
// simliIdle = false; simliIdle = false;
// simliGeneration += 1; simliGeneration += 1;
if (micResumeTimer) { if (micResumeTimer) {
clearTimeout(micResumeTimer); clearTimeout(micResumeTimer);
micResumeTimer = 0; micResumeTimer = 0;
} }
// disconnectSimliTts(); disconnectSimliTts();
stopWaveLoop(); stopWaveLoop();
userAnalyser = null; userAnalyser = null;
agentAnalyser = null; agentAnalyser = null;
@ -933,8 +812,8 @@ async function teardownMedia() {
micPublished = false; micPublished = false;
micListening = true; micListening = true;
lastAgentMediaTrack = null; lastAgentMediaTrack = null;
// stopAgentTrackClone(); stopAgentTrackClone();
// pipedTrackId = null; pipedTrackId = null;
mutedLivekitAudioEls.splice(0).forEach((node) => { mutedLivekitAudioEls.splice(0).forEach((node) => {
try { try {
if (node.srcObject) node.srcObject = null; if (node.srcObject) node.srcObject = null;
@ -952,33 +831,15 @@ async function teardownMedia() {
/* ignore */ /* ignore */
} }
} }
// const simli = simliClient; const simli = simliClient;
// simliClient = null; simliClient = null;
// if (simli && typeof simli.stop === 'function') { if (simli && typeof simli.stop === 'function') {
// try {
// await simli.stop();
// } catch {
// /* ignore */
// }
// }
if (avatarAudioSource) {
try { try {
avatarAudioSource.disconnect(); await simli.stop();
} catch { } catch {
// ignore /* ignore */
} }
}
avatarAudioSource = null;
}
avatarAudioAnalyser = null;
if (
window.EBAvatar3D &&
typeof window.EBAvatar3D.destroy === 'function'
) {
window.EBAvatar3D.destroy();
}
} }
async function maybeResume() { async function maybeResume() {
@ -1018,34 +879,11 @@ async function loadChat() {
if (!config.voiceEnabled) { if (!config.voiceEnabled) {
throw new Error('Voice is not enabled for this chatbot'); throw new Error('Voice is not enabled for this chatbot');
} }
videoAvatarOn = Boolean( videoAvatarOn = Boolean(config.videoAvatarEnabled || config.avatarEnabled);
config.videoAvatarEnabled || applyStageMode();
config.avatarEnabled
);
if (videoAvatarOn) {
if (
!window.EBAvatar3D ||
typeof window.EBAvatar3D.init !== 'function'
) {
throw new Error(
'3D avatar client failed to load'
);
}
await window.EBAvatar3D.init({
container: avatarStageEl,
modelUrl: '/embed/avatar/model.glb',
});
console.info(
'[avatar3d] avatar initialized'
);
}
} }
async function startSession() { async function startSession() {
console.log('[DEBUG] startSession() START');
if (started) return; if (started) return;
started = true; started = true;
hideStartOverlay(); hideStartOverlay();
@ -1060,11 +898,6 @@ async function startSession() {
} }
await loadChat(); await loadChat();
console.log('[DEBUG] loadChat() DONE', {
videoAvatarOn,
avatarStageEl: !!avatarStageEl,
EBAvatar3D: !!window.EBAvatar3D
});
if (!navigator.mediaDevices?.getUserMedia) { if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('This browser cannot access the microphone'); throw new Error('This browser cannot access the microphone');
@ -1079,7 +912,7 @@ async function startSession() {
sessionId = granted.sessionId || sessionId; sessionId = granted.sessionId || sessionId;
sessionAlive = true; sessionAlive = true;
// if (!videoAvatarOn) startWaveLoop(); if (!videoAvatarOn) startWaveLoop();
const room = new Livekit.Room({ const room = new Livekit.Room({
adaptiveStream: false, adaptiveStream: false,
@ -1099,13 +932,7 @@ async function startSession() {
setStatus('Disconnected'); setStatus('Disconnected');
} }
}); });
room.on(Livekit.RoomEvent.TrackSubscribed, (track, pub, participant) => { room.on(Livekit.RoomEvent.TrackSubscribed, (track, _pub, participant) => {
console.log('[DEBUG] TrackSubscribed', {
kind: track?.kind,
source: pub?.source,
participant: participant?.identity,
trackSid: pub?.trackSid
});
handleIncomingAudio(track, participant, room); handleIncomingAudio(track, participant, room);
}); });
if (Livekit.RoomEvent.LocalTrackPublished) { if (Livekit.RoomEvent.LocalTrackPublished) {
@ -1132,9 +959,8 @@ async function startSession() {
const local = isLocalParticipant(room, participant); const local = isLocalParticipant(room, participant);
(segments || []).forEach((seg) => { (segments || []).forEach((seg) => {
if (!seg || !seg.final) return; if (!seg || !seg.final) return;
if (agentPipelineState === 'speaking' && local) { if (agentPipelineState === 'speaking' && local) return;
return; ensureSimliConnected(local ? 'user-speech' : 'agent-speech');
}
if (rememberTranscript(seg.id || `${local ? 'user' : 'bot'}:${seg.text}`)) return; if (rememberTranscript(seg.id || `${local ? 'user' : 'bot'}:${seg.text}`)) return;
if (!local && pendingGreeting && normalizeSpeech(seg.text) === normalizeSpeech(pendingGreeting)) { if (!local && pendingGreeting && normalizeSpeech(seg.text) === normalizeSpeech(pendingGreeting)) {
showPendingGreeting(); showPendingGreeting();
@ -1144,67 +970,17 @@ async function startSession() {
}); });
}); });
room.on(Livekit.RoomEvent.ActiveSpeakersChanged, (speakers) => { room.on(Livekit.RoomEvent.ActiveSpeakersChanged, (speakers) => {
if (videoAvatarOn) return; if ((speakers || []).length) ensureSimliConnected('speakers');
if (simliIdle || simliRestarting || agentPipelineState === 'speaking' || agentPipelineState === 'thinking') {
const localId = room.localParticipant?.identity; return;
const agentSpeaking = (speakers || []).some(
(p) => p?.identity && p.identity !== localId
);
setStatus(agentSpeaking ? 'Speaking' : 'Listening');
});
room.on(Livekit.RoomEvent.ParticipantConnected, (participant) => {
console.log('[DEBUG] ParticipantConnected', {
identity: participant?.identity,
sid: participant?.sid,
});
participant.trackPublications.forEach((pub) => {
console.log('[DEBUG] Remote publication', {
kind: pub?.kind,
source: pub?.source,
trackSid: pub?.trackSid,
subscribed: pub?.isSubscribed,
hasTrack: !!pub?.track,
});
if (pub?.track) {
handleIncomingAudio(pub.track, participant, room);
} }
if (videoAvatarOn) return;
const localId = room.localParticipant?.identity;
const agentSpeaking = (speakers || []).some((p) => p?.identity && p.identity !== localId);
setStatus(agentSpeaking ? 'Speaking' : 'Listening');
}); });
});
room.on(Livekit.RoomEvent.ParticipantConnected, (participant) => {
console.log('[DEBUG] ParticipantConnected', {
identity: participant?.identity,
sid: participant?.sid,
});
});
room.on(Livekit.RoomEvent.ParticipantDisconnected, (participant) => {
console.log('[DEBUG] ParticipantDisconnected', {
identity: participant?.identity,
sid: participant?.sid,
});
});
room.on(Livekit.RoomEvent.LocalTrackPublished, (publication) => {
console.log('[DEBUG] LocalTrackPublished', {
kind: publication?.kind,
source: publication?.source,
trackSid: publication?.trackSid,
});
});
await room.connect(granted.url, granted.token); await room.connect(granted.url, granted.token);
console.log('[DEBUG] LiveKit CONNECTED', {
localParticipant: room.localParticipant?.identity,
remoteParticipants: room.remoteParticipants?.size
});
if (typeof room.startAudio === 'function') { if (typeof room.startAudio === 'function') {
await room.startAudio().catch(() => {}); await room.startAudio().catch(() => {});
} }
@ -1228,11 +1004,6 @@ room.on(Livekit.RoomEvent.LocalTrackPublished, (publication) => {
} }
if (!micPub) throw new Error('Microphone did not publish. Allow mic access and try again.'); if (!micPub) throw new Error('Microphone did not publish. Allow mic access and try again.');
micPublished = true; micPublished = true;
console.log('[DEBUG] MIC PUBLISHED', {
trackSid: micPub?.trackSid,
kind: micPub?.kind,
source: micPub?.source,
});
micListening = true; micListening = true;
const localMedia = mediaTrackFrom(micPub.track); const localMedia = mediaTrackFrom(micPub.track);
if (localMedia) hookWaveAnalyser(localMedia, 'user'); if (localMedia) hookWaveAnalyser(localMedia, 'user');
@ -1247,25 +1018,17 @@ room.on(Livekit.RoomEvent.LocalTrackPublished, (publication) => {
setStatus('Listening'); setStatus('Listening');
return; return;
} }
// setStatus('Connecting avatar…'); setStatus('Connecting avatar…');
// try { try {
// await Promise.race([ await Promise.race([
// startSimliOverlay(), startSimliOverlay(),
// new Promise((resolve) => setTimeout(resolve, 2500)), new Promise((resolve) => setTimeout(resolve, 2500)),
// ]); ]);
// } catch (err) { } catch (err) {
// console.warn('[avatar] simli overlay', err); console.warn('[avatar] simli overlay', err);
// } }
// await signalAvatarReady(room); await signalAvatarReady(room);
// applyAgentState(readRemoteAgentState(room) || agentPipelineState); applyAgentState(readRemoteAgentState(room) || agentPipelineState);
setStatus('Listening');
await signalAvatarReady(room);
applyAgentState(
readRemoteAgentState(room) ||
agentPipelineState
);
} }
function needsGesture(err) { function needsGesture(err) {

646
public/embed/avatar/avatar3d.js
View File

@ -1,646 +0,0 @@
(function () {
'use strict';
let THREE = null;
let GLTFLoader = null;
let scene = null;
let camera = null;
let renderer = null;
let model = null;
let container = null;
let animationFrame = 0;
let analyser = null;
let frequencyData = null;
const morphs = {
jawOpen: [],
mouthClose: [],
sil: [],
};
let initialized = false;
// --------------------------------------------------
// Load Three.js + GLTFLoader
// --------------------------------------------------
async function loadThree() {
if (THREE && GLTFLoader) {
return;
}
const threeModule = await import(
'https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js'
);
THREE = threeModule;
const loaderModule = await import(
'https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/loaders/GLTFLoader.js'
);
GLTFLoader = loaderModule.GLTFLoader;
}
// --------------------------------------------------
// Initialize
// --------------------------------------------------
async function init(options = {}) {
if (initialized) {
return window.EBAvatar3D;
}
container =
options.container ||
document.getElementById('eba-avatar-stage');
if (!container) {
throw new Error(
'[avatar3d] #eba-avatar-stage not found'
);
}
await loadThree();
createRenderer();
createScene();
createCamera();
createLights();
await loadModel(
options.modelUrl || '/embed/avatar/model.glb'
);
setupResize();
initialized = true;
startRenderLoop();
console.info('[avatar3d] initialized');
return window.EBAvatar3D;
}
// --------------------------------------------------
// Renderer
// --------------------------------------------------
function createRenderer() {
renderer = new THREE.WebGLRenderer({
antialias: true,
alpha: false,
powerPreference: 'high-performance',
});
renderer.setPixelRatio(
Math.min(window.devicePixelRatio || 1, 2)
);
renderer.setSize(
container.clientWidth || 640,
container.clientHeight || 640,
false
);
renderer.outputColorSpace = THREE.SRGBColorSpace;
renderer.domElement.id = 'eba-avatar-canvas';
renderer.domElement.style.display = 'block';
renderer.domElement.style.width = '100%';
renderer.domElement.style.height = '100%';
container.appendChild(renderer.domElement);
}
// --------------------------------------------------
// Scene
// --------------------------------------------------
function createScene() {
scene = new THREE.Scene();
scene.background = new THREE.Color(0x020617);
}
// --------------------------------------------------
// Camera
// --------------------------------------------------
function createCamera() {
const width =
container.clientWidth || 640;
const height =
container.clientHeight || 640;
camera = new THREE.PerspectiveCamera(
30,
width / height,
0.01,
100
);
camera.position.set(
0,
1.55,
2.7
);
camera.lookAt(
0,
1.45,
0
);
}
// --------------------------------------------------
// Lights
// --------------------------------------------------
function createLights() {
const ambient =
new THREE.HemisphereLight(
0xffffff,
0x223344,
2
);
scene.add(ambient);
const key =
new THREE.DirectionalLight(
0xffffff,
2.5
);
key.position.set(2, 3, 4);
scene.add(key);
const fill =
new THREE.DirectionalLight(
0xaac8ff,
1
);
fill.position.set(-3, 2, 2);
scene.add(fill);
}
// --------------------------------------------------
// Load GLB
// --------------------------------------------------
async function loadModel(url) {
return new Promise((resolve, reject) => {
const loader = new GLTFLoader();
console.info(
'[avatar3d] loading model:',
url
);
loader.load(
url,
(gltf) => {
model = gltf.scene;
scene.add(model);
inspectModel(model);
fitModel();
console.info(
'[avatar3d] model loaded'
);
resolve(model);
},
(progress) => {
if (progress.total) {
const percent =
(progress.loaded /
progress.total) *
100;
console.info(
`[avatar3d] loading ${percent.toFixed(0)}%`
);
}
},
(error) => {
console.error(
'[avatar3d] model loading failed',
error
);
reject(error);
}
);
});
}
// --------------------------------------------------
// Find morph targets
// --------------------------------------------------
function inspectModel(root) {
Object.keys(morphs).forEach((key) => {
morphs[key].length = 0;
});
root.traverse((object) => {
if (!object.isMesh) {
return;
}
if (
!object.morphTargetDictionary ||
!object.morphTargetInfluences
) {
return;
}
console.info(
'[avatar3d] morph mesh:',
object.name
);
registerMorph(
object,
'jawOpen'
);
registerMorph(
object,
'mouthClose'
);
registerMorph(
object,
'sil'
);
});
console.info(
'[avatar3d] jawOpen targets:',
morphs.jawOpen.length
);
}
function registerMorph(mesh, name) {
const dictionary =
mesh.morphTargetDictionary;
if (
dictionary &&
Object.prototype.hasOwnProperty.call(
dictionary,
name
)
) {
morphs[name].push({
mesh,
index: dictionary[name],
});
}
}
// --------------------------------------------------
// Fit model
// --------------------------------------------------
function fitModel() {
if (!model) {
return;
}
const box =
new THREE.Box3().setFromObject(model);
const size =
new THREE.Vector3();
const center =
new THREE.Vector3();
box.getSize(size);
box.getCenter(center);
console.info(
'[avatar3d] model size:',
size
);
model.position.x -= center.x;
model.position.y -= box.min.y;
model.position.z -= center.z;
const maxDimension =
Math.max(
size.x,
size.y,
size.z
);
if (maxDimension > 5) {
const scale =
2 / maxDimension;
model.scale.setScalar(scale);
}
}
// --------------------------------------------------
// Audio analyser
// --------------------------------------------------
function setAudioAnalyser(nextAnalyser) {
analyser = nextAnalyser;
if (analyser) {
frequencyData =
new Uint8Array(
analyser.frequencyBinCount
);
console.info(
'[avatar3d] audio analyser connected'
);
} else {
frequencyData = null;
console.info(
'[avatar3d] audio analyser disconnected'
);
}
}
function getAudioLevel() {
if (!analyser || !frequencyData) {
return 0;
}
analyser.getByteFrequencyData(
frequencyData
);
let sum = 0;
for (
let i = 0;
i < frequencyData.length;
i++
) {
const value =
frequencyData[i] / 255;
sum += value * value;
}
const rms =
Math.sqrt(
sum /
frequencyData.length
);
return Math.min(
1,
rms * 2.5
);
}
// --------------------------------------------------
// Lip movement
// --------------------------------------------------
function updateMouth() {
const level =
getAudioLevel();
const target =
Math.min(
1,
Math.max(
0,
level * 2.2
)
);
// Jaw opens when agent speaks.
morphs.jawOpen.forEach(
(targetInfo) => {
const current =
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] || 0;
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] =
THREE.MathUtils.lerp(
current,
target,
0.35
);
}
);
// Mouth close when silent.
morphs.mouthClose.forEach(
(targetInfo) => {
const current =
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] || 0;
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] =
THREE.MathUtils.lerp(
current,
Math.max(
0,
1 - target
),
0.15
);
}
);
// Silence target.
morphs.sil.forEach(
(targetInfo) => {
const current =
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] || 0;
const silence =
target < 0.03
? 1
: 0;
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] =
THREE.MathUtils.lerp(
current,
silence,
0.15
);
}
);
}
// --------------------------------------------------
// Render loop
// --------------------------------------------------
function startRenderLoop() {
if (animationFrame) {
return;
}
const render = () => {
animationFrame =
requestAnimationFrame(
render
);
updateMouth();
if (
renderer &&
scene &&
camera
) {
renderer.render(
scene,
camera
);
}
};
render();
}
// --------------------------------------------------
// Resize
// --------------------------------------------------
function setupResize() {
window.addEventListener(
'resize',
resize,
{ passive: true }
);
resize();
}
function resize() {
if (
!renderer ||
!camera ||
!container
) {
return;
}
const width =
container.clientWidth || 640;
const height =
container.clientHeight || 640;
camera.aspect =
width / height;
camera.updateProjectionMatrix();
renderer.setSize(
width,
height,
false
);
}
// --------------------------------------------------
// Destroy
// --------------------------------------------------
function destroy() {
if (animationFrame) {
cancelAnimationFrame(
animationFrame
);
animationFrame = 0;
}
if (renderer) {
renderer.dispose();
if (
renderer.domElement &&
renderer.domElement.parentNode
) {
renderer.domElement.parentNode.removeChild(
renderer.domElement
);
}
}
scene = null;
camera = null;
renderer = null;
model = null;
analyser = null;
frequencyData = null;
Object.keys(morphs).forEach(
(key) => {
morphs[key].length = 0;
}
);
initialized = false;
}
// --------------------------------------------------
// Public API
// --------------------------------------------------
window.EBAvatar3D = {
init,
setAudioAnalyser,
destroy,
get initialized() {
return initialized;
},
get model() {
return model;
},
};
})();

BIN
public/embed/avatar/model.glb
View File

Binary file not shown.

98
public/embed/simli-client.iife.js
View File

File diff suppressed because one or more lines are too long