Stratawiz
/
Eventbuzz_Concierge
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update Concierge integration

This commit is contained in:
Tanuja Pansare 2026-10-19 12:51:48 +05:30
parent 026ea7d0d2
commit 6c80a74ee5
11 changed files with 1329 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
public/embed-avatar.html
View File

@ -9,10 +9,11 @@
<body> <body>
<div id="eba-root"> <div id="eba-root">
<section class="eba-avatar" aria-label="Concierge avatar"> <section class="eba-avatar" aria-label="Concierge avatar">
<video id="eba-video" autoplay playsinline muted></video> <div id="eba-avatar-stage"></div>
<audio id="eba-audio" autoplay playsinline></audio>
<canvas id="eba-wave" aria-hidden="true"></canvas> <audio id="eba-audio" autoplay playsinline></audio>
<div id="eba-avatar-status">Starting…</div>
<div id="eba-avatar-status">Starting…</div>
</section> </section>
<section class="eba-chat" aria-label="Concierge chat"> <section class="eba-chat" aria-label="Concierge chat">
<header class="eba-chat-header"> <header class="eba-chat-header">
@ -42,6 +43,14 @@
}; };
</script> </script>
<script src="https://cdn.jsdelivr.net/npm/livekit-client@2.15.14/dist/livekit-client.umd.js"></script> <script src="https://cdn.jsdelivr.net/npm/livekit-client@2.15.14/dist/livekit-client.umd.js"></script>
<script type="module" src="/embed/avatar-page.js?v=9"></script> <script src="/embed/avatar/avatar3d.js"></script>
<script type="importmap">
{
"imports": {
"three": "https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js"
}
}
</script>
<script type="module" src="/embed/avatar-page.js?v=11"></script>
</body> </body>
</html> </html>

33
public/embed/avatar-page.css
View File

@ -29,17 +29,32 @@ body {
} }
.eba-avatar { .eba-avatar {
position: relative; position: relative;
background: #020617; background: #020617;
min-width: 0; min-width: 0;
min-height: 0; min-height: 0;
overflow: hidden;
} }
#eba-video { #eba-avatar-stage {
width: 100%; position: absolute;
height: 100%; inset: 0;
object-fit: cover; width: 100%;
background: #020617; height: 100%;
}
#eba-avatar-stage canvas {
display: block;
width: 100%;
height: 100%;
}
#eba-audio {
position: absolute;
width: 1px;
height: 1px;
opacity: 0;
pointer-events: none;
} }
#eba-audio { #eba-audio {

619
public/embed/avatar-page.js
View File

@ -4,8 +4,10 @@ const apiBase = String(cfg.apiBase || '').replace(/\/$/, '');
const Livekit = window.LivekitClient || window.livekit; const Livekit = window.LivekitClient || window.livekit;
const videoEl = document.getElementById('eba-video'); const videoEl = document.getElementById('eba-video');
// const audioEl = document.getElementById('eba-audio');
// const waveEl = document.getElementById('eba-wave');
const audioEl = document.getElementById('eba-audio'); const audioEl = document.getElementById('eba-audio');
const waveEl = document.getElementById('eba-wave'); const avatarStageEl = document.getElementById('eba-avatar-stage');
const messagesEl = document.getElementById('eba-messages'); const messagesEl = document.getElementById('eba-messages');
const errorEl = document.getElementById('eba-error'); const errorEl = document.getElementById('eba-error');
const formEl = document.getElementById('eba-form'); const formEl = document.getElementById('eba-form');
@ -20,10 +22,10 @@ const subEl = document.getElementById('eba-sub');
let sessionId = null; let sessionId = null;
let busy = false; let busy = false;
let livekitRoom = null; let livekitRoom = null;
let simliClient = null; // let simliClient = null;
let started = false; let started = false;
let chatReady = false; let chatReady = false;
let pipedTrackId = null; // let pipedTrackId = null;
let seenTranscriptIds = {}; let seenTranscriptIds = {};
let chipsEl = null; let chipsEl = null;
const mutedLivekitAudioEls = []; const mutedLivekitAudioEls = [];
@ -32,22 +34,24 @@ let pendingQuickReplies = [];
let greetingShown = false; let greetingShown = false;
let avatarReadySent = false; let avatarReadySent = false;
let lastAgentMediaTrack = null; let lastAgentMediaTrack = null;
let lastAgentTrackClone = null; // let lastAgentTrackClone = null;
let sessionAlive = false; let sessionAlive = false;
let simliRestarting = false; // let simliRestarting = false;
let simliIdle = false; // let simliIdle = false;
let simliGeneration = 0; // let simliGeneration = 0;
let videoAvatarOn = false; let videoAvatarOn = false;
let waveAudioCtx = null; let waveAudioCtx = null;
let userAnalyser = null; let userAnalyser = null;
let agentAnalyser = null; let agentAnalyser = null;
let avatarAudioSource = null;
let avatarAudioAnalyser = null;
let waveRaf = 0; let waveRaf = 0;
let simliInputEnabled = false; // let simliInputEnabled = false;
let agentPipelineState = ''; let agentPipelineState = '';
let micListening = true; let micListening = true;
let micPublished = false; let micPublished = false;
let micResumeTimer = 0; let micResumeTimer = 0;
const SIMLI_AUDIO_BUFFER = 3000; // const SIMLI_AUDIO_BUFFER = 3000;
const AGENT_STATE_ATTR = 'lk.agent.state'; const AGENT_STATE_ATTR = 'lk.agent.state';
function setStatus(text) { function setStatus(text) {
@ -259,6 +263,10 @@ async function signalAvatarReady(room) {
try { try {
if (typeof room.localParticipant.publishData === 'function') { if (typeof room.localParticipant.publishData === 'function') {
await room.localParticipant.publishData(payload, { reliable: true }); await room.localParticipant.publishData(payload, { reliable: true });
console.log('[DEBUG] avatar-ready SENT', {
identity: room.localParticipant.identity,
room: room.name,
});
} }
} catch (err) { } catch (err) {
console.warn('[avatar] ready signal failed', err); console.warn('[avatar] ready signal failed', err);
@ -306,7 +314,7 @@ async function sendMessage() {
inputEl.value = ''; inputEl.value = '';
renderChips([]); renderChips([]);
addMessage('user', text); addMessage('user', text);
ensureSimliConnected('chat'); // ensureSimliConnected('chat');
setBusy(true); setBusy(true);
try { try {
const data = await api(`/public/concierge/${encodeURIComponent(botId)}/chat`, { const data = await api(`/public/concierge/${encodeURIComponent(botId)}/chat`, {
@ -445,28 +453,28 @@ async function setMicListening(on) {
function applyAgentState(state) { function applyAgentState(state) {
const next = String(state || '').toLowerCase(); const next = String(state || '').toLowerCase();
if (!next) return; if (!next) return;
agentPipelineState = next; agentPipelineState = next;
const speaking = next === 'speaking'; const speaking = next === 'speaking';
const holdMic = speaking || next === 'thinking'; const holdMic = speaking || next === 'thinking';
if (videoAvatarOn) {
if (speaking && lastAgentMediaTrack) {
if (!simliInputEnabled) connectSimliTts(lastAgentMediaTrack);
} else if (!speaking) {
disconnectSimliTts();
}
}
if (holdMic) { if (holdMic) {
setMicListening(false); setMicListening(false);
} else { } else {
micResumeTimer = window.setTimeout(() => setMicListening(true), 700); micResumeTimer = window.setTimeout(
() => setMicListening(true),
700
);
} }
if (simliIdle || simliRestarting) return; if (next === 'thinking') {
if (next === 'thinking') setStatus('Thinking'); setStatus('Thinking');
else if (speaking) setStatus('Speaking'); } else if (speaking) {
else setStatus('Listening'); setStatus('Speaking');
} else {
setStatus('Listening');
}
} }
function makeSimliClient(SimliClient, LogLevel, token) { function makeSimliClient(SimliClient, LogLevel, token) {
@ -544,24 +552,106 @@ function ensureWaveAudioCtx() {
if (waveAudioCtx.state === 'suspended') waveAudioCtx.resume().catch(() => {}); if (waveAudioCtx.state === 'suspended') waveAudioCtx.resume().catch(() => {});
return waveAudioCtx; return waveAudioCtx;
} }
function hookAvatarAnalyser(mediaTrack) {
if (!mediaTrack) {
return;
}
const ctx = ensureWaveAudioCtx();
if (!ctx) {
console.warn(
'[avatar3d] AudioContext unavailable'
);
return;
}
try {
if (avatarAudioSource) {
try {
avatarAudioSource.disconnect();
} catch {
// ignore
}
avatarAudioSource = null;
}
const clone =
typeof mediaTrack.clone === 'function'
? mediaTrack.clone()
: mediaTrack;
const stream =
new MediaStream([clone]);
avatarAudioSource =
ctx.createMediaStreamSource(
stream
);
avatarAudioAnalyser =
ctx.createAnalyser();
avatarAudioAnalyser.fftSize = 256;
avatarAudioAnalyser.smoothingTimeConstant = 0.72;
avatarAudioSource.connect(
avatarAudioAnalyser
);
if (
window.EBAvatar3D &&
typeof window.EBAvatar3D.setAudioAnalyser ===
'function'
) {
window.EBAvatar3D.setAudioAnalyser(
avatarAudioAnalyser
);
}
console.info(
'[avatar3d] agent audio connected to avatar'
);
} catch (err) {
console.warn(
'[avatar3d] audio analyser failed',
err
);
}
}
function hookWaveAnalyser(mediaTrack, slot) { function hookWaveAnalyser(mediaTrack, slot) {
if (!mediaTrack || videoAvatarOn) return; if (!mediaTrack || videoAvatarOn) return;
const ctx = ensureWaveAudioCtx(); const ctx = ensureWaveAudioCtx();
if (!ctx) return; if (!ctx) return;
try { try {
const clone = typeof mediaTrack.clone === 'function' ? mediaTrack.clone() : mediaTrack; const clone =
const source = ctx.createMediaStreamSource(new MediaStream([clone])); typeof mediaTrack.clone === 'function'
? mediaTrack.clone()
: mediaTrack;
const source = ctx.createMediaStreamSource(
new MediaStream([clone])
);
const analyser = ctx.createAnalyser(); const analyser = ctx.createAnalyser();
analyser.fftSize = 256; analyser.fftSize = 256;
analyser.smoothingTimeConstant = 0.72; analyser.smoothingTimeConstant = 0.72;
source.connect(analyser); source.connect(analyser);
if (slot === 'user') userAnalyser = analyser;
else agentAnalyser = analyser; if (slot === 'user') {
userAnalyser = analyser;
} else {
agentAnalyser = analyser;
}
} catch (err) { } catch (err) {
console.warn('[avatar] wave analyser failed', err); console.warn('[avatar] wave analyser failed', err);
} }
startWaveLoop();
} }
function analyserSnapshot(analyser) { function analyserSnapshot(analyser) {
@ -656,150 +746,181 @@ function applyStageMode() {
} }
function handleIncomingAudio(track, participant, room) { function handleIncomingAudio(track, participant, room) {
if (!track || String(track.kind || '').toLowerCase() !== 'audio') return; console.log('[DEBUG] handleIncomingAudio()', {
if (isLocalParticipant(room, participant)) return; kind: track?.kind,
attachRemoteAudio(track, true); participant: participant?.identity,
const media = mediaTrackFrom(track); isLocal: isLocalParticipant(room, participant)
lastAgentMediaTrack = media; });
if (videoAvatarOn) { if (
if (agentPipelineState === 'speaking') connectSimliTts(media); !track ||
} else { String(track.kind || '').toLowerCase() !== 'audio'
hookWaveAnalyser(media, 'agent'); ) {
setTimeout(() => signalAvatarReady(room), 400); return;
}
}
function ensureSimliConnected(reason) {
if (!videoAvatarOn) return;
if (!sessionAlive || !livekitRoom || simliRestarting) return;
if (!simliIdle && simliClient) return;
restartSimliAvatar(reason || 'session-restart');
}
async function restartSimliAvatar(reason) {
if (!sessionAlive || simliRestarting || !livekitRoom) return;
simliRestarting = true;
simliGeneration += 1;
setStatus('Reconnecting avatar…');
console.warn('[avatar] restarting simli', reason);
try {
const prev = simliClient;
simliClient = null;
if (prev && typeof prev.stop === 'function') {
await prev.stop().catch(() => {});
} }
const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, {
method: 'POST',
body: {},
});
if (!simli?.sessionToken) throw new Error('Avatar session token was empty');
const { SimliClient, LogLevel } = await loadSimliClient();
const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken);
bindSimliEvents(next);
simliClient = next;
await next.start();
const played = await resumeSimliAudio();
pipedTrackId = null;
if (played) muteLivekitPlayback();
else unmuteLivekitFallback();
simliIdle = false;
applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
} catch (err) {
console.warn('[avatar] simli restart failed', err);
simliIdle = true;
setStatus('Avatar disconnected');
} finally {
simliRestarting = false;
}
}
function bindSimliEvents(client) { if (isLocalParticipant(room, participant)) {
if (!client || typeof client.on !== 'function') return; return;
const gen = simliGeneration; }
let ready = false;
client.on('start', () => { // --------------------------------------------
if (gen !== simliGeneration) return; // Authoritative audio playback
ready = true; // --------------------------------------------
simliIdle = false; console.log('[DEBUG] Attaching agent audio');
muteLivekitPlayback(); attachRemoteAudio(track, true);
unmuteSimliPlayback();
if (audioEl?.play) audioEl.play().catch(() => {}); // --------------------------------------------
applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState); // Same audio -> 3D avatar
}); // --------------------------------------------
client.on('stop', () => {
if (gen !== simliGeneration) return; const media = mediaTrackFrom(track);
simliIdle = true; console.log('[DEBUG] handleIncomingAudio()', {
setStatus('Idle — speak to continue'); kind: track?.kind,
}); participant: participant?.identity,
client.on('error', (message) => { isLocal: isLocalParticipant(room, participant)
if (gen !== simliGeneration) return;
console.warn('[avatar] simli error', message);
if (sessionAlive && ready) restartSimliAvatar(message || 'error');
}); });
lastAgentMediaTrack = media;
if (media) {
hookAvatarAnalyser(media);
}
setTimeout(() => {
signalAvatarReady(room);
}, 400);
} }
async function loadSimliClient() { // function ensureSimliConnected(reason) {
const mod = await import('/embed/simli-client.js'); // if (!videoAvatarOn) return;
const SimliClient = mod.SimliClient || mod.default?.SimliClient || mod.default; // if (!sessionAlive || !livekitRoom || simliRestarting) return;
if (typeof SimliClient !== 'function') { // if (!simliIdle && simliClient) return;
throw new Error('SIMLI client failed to load'); // restartSimliAvatar(reason || 'session-restart');
} // }
return { SimliClient, LogLevel: mod.LogLevel || mod.default?.LogLevel };
}
async function resumeSimliAudio() { // async function restartSimliAvatar(reason) {
const ctx = simliClient?.audioContext; // if (!sessionAlive || simliRestarting || !livekitRoom) return;
if (ctx && ctx.state === 'suspended' && typeof ctx.resume === 'function') { // simliRestarting = true;
await ctx.resume().catch(() => {}); // // simliGeneration += 1;
} // setStatus('Reconnecting avatar…');
unmuteSimliPlayback(); // console.warn('[avatar] restarting simli', reason);
let played = false; // try {
if (audioEl?.play) { // const prev = simliClient;
played = await audioEl.play().then(() => true).catch(() => false); // simliClient = null;
} // if (prev && typeof prev.stop === 'function') {
if (videoEl?.play) videoEl.play().catch(() => {}); // await prev.stop().catch(() => {});
return played; // }
} // const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, {
// method: 'POST',
// body: {},
// });
// if (!simli?.sessionToken) throw new Error('Avatar session token was empty');
// const { SimliClient, LogLevel } = await loadSimliClient();
// const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken);
// bindSimliEvents(next);
// simliClient = next;
// await next.start();
// const played = await resumeSimliAudio();
// pipedTrackId = null;
// if (played) muteLivekitPlayback();
// else unmuteLivekitFallback();
// // simliIdle = false;
// applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
// } catch (err) {
// console.warn('[avatar] simli restart failed', err);
// simliIdle = true;
// setStatus('Avatar disconnected');
// } finally {
// simliRestarting = false;
// }
// }
async function startSimliOverlay() { // function bindSimliEvents(client) {
if (!videoAvatarOn || simliRestarting) return; // if (!client || typeof client.on !== 'function') return;
simliRestarting = true; // const gen = simliGeneration;
try { // let ready = false;
setStatus('Connecting avatar…'); // client.on('start', () => {
const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, { // if (gen !== simliGeneration) return;
method: 'POST', // ready = true;
body: {}, // // simliIdle = false;
}); // muteLivekitPlayback();
if (!simli?.sessionToken) throw new Error('Avatar session token was empty'); // unmuteSimliPlayback();
const { SimliClient, LogLevel } = await loadSimliClient(); // if (audioEl?.play) audioEl.play().catch(() => {});
const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken); // applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
bindSimliEvents(next); // });
simliClient = next; // client.on('stop', () => {
await next.start(); // if (gen !== simliGeneration) return;
const played = await resumeSimliAudio(); // simliIdle = true;
pipedTrackId = null; // setStatus('Idle — speak to continue');
if (played) muteLivekitPlayback(); // });
else unmuteLivekitFallback(); // client.on('error', (message) => {
applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState); // if (gen !== simliGeneration) return;
} catch (err) { // console.warn('[avatar] simli error', message);
console.warn('[avatar] simli overlay failed, voice continues', err); // if (sessionAlive && ready) restartSimliAvatar(message || 'error');
simliClient = null; // });
unmuteLivekitFallback(); // }
setStatus('Voice only');
} finally { // async function loadSimliClient() {
simliRestarting = false; // const mod = await import('/embed/simli-client.js');
} // const SimliClient = mod.SimliClient || mod.default?.SimliClient || mod.default;
} // if (typeof SimliClient !== 'function') {
// throw new Error('SIMLI client failed to load');
// }
// return { SimliClient, LogLevel: mod.LogLevel || mod.default?.LogLevel };
// }
// async function resumeSimliAudio() {
// const ctx = simliClient?.audioContext;
// if (ctx && ctx.state === 'suspended' && typeof ctx.resume === 'function') {
// await ctx.resume().catch(() => {});
// }
// unmuteSimliPlayback();
// let played = false;
// if (audioEl?.play) {
// played = await audioEl.play().then(() => true).catch(() => false);
// }
// if (videoEl?.play) videoEl.play().catch(() => {});
// return played;
// }
// async function startSimliOverlay() {
// if (!videoAvatarOn || simliRestarting) return;
// simliRestarting = true;
// try {
// setStatus('Connecting avatar…');
// const simli = await api(`/public/concierge/${encodeURIComponent(botId)}/simli/token`, {
// method: 'POST',
// body: {},
// });
// if (!simli?.sessionToken) throw new Error('Avatar session token was empty');
// const { SimliClient, LogLevel } = await loadSimliClient();
// const next = makeSimliClient(SimliClient, LogLevel, simli.sessionToken);
// bindSimliEvents(next);
// simliClient = next;
// await next.start();
// const played = await resumeSimliAudio();
// pipedTrackId = null;
// if (played) muteLivekitPlayback();
// else unmuteLivekitFallback();
// applyAgentState(readRemoteAgentState(livekitRoom) || agentPipelineState);
// } catch (err) {
// console.warn('[avatar] simli overlay failed, voice continues', err);
// simliClient = null;
// unmuteLivekitFallback();
// setStatus('Voice only');
// } finally {
// simliRestarting = false;
// }
// }
async function teardownMedia() { async function teardownMedia() {
sessionAlive = false; sessionAlive = false;
simliIdle = false; // simliIdle = false;
simliGeneration += 1; // simliGeneration += 1;
if (micResumeTimer) { if (micResumeTimer) {
clearTimeout(micResumeTimer); clearTimeout(micResumeTimer);
micResumeTimer = 0; micResumeTimer = 0;
} }
disconnectSimliTts(); // disconnectSimliTts();
stopWaveLoop(); stopWaveLoop();
userAnalyser = null; userAnalyser = null;
agentAnalyser = null; agentAnalyser = null;
@ -812,8 +933,8 @@ async function teardownMedia() {
micPublished = false; micPublished = false;
micListening = true; micListening = true;
lastAgentMediaTrack = null; lastAgentMediaTrack = null;
stopAgentTrackClone(); // stopAgentTrackClone();
pipedTrackId = null; // pipedTrackId = null;
mutedLivekitAudioEls.splice(0).forEach((node) => { mutedLivekitAudioEls.splice(0).forEach((node) => {
try { try {
if (node.srcObject) node.srcObject = null; if (node.srcObject) node.srcObject = null;
@ -831,15 +952,33 @@ async function teardownMedia() {
/* ignore */ /* ignore */
} }
} }
const simli = simliClient; // const simli = simliClient;
simliClient = null; // simliClient = null;
if (simli && typeof simli.stop === 'function') { // if (simli && typeof simli.stop === 'function') {
// try {
// await simli.stop();
// } catch {
// /* ignore */
// }
// }
if (avatarAudioSource) {
try { try {
await simli.stop(); avatarAudioSource.disconnect();
} catch { } catch {
/* ignore */ // ignore
} }
}
avatarAudioSource = null;
}
avatarAudioAnalyser = null;
if (
window.EBAvatar3D &&
typeof window.EBAvatar3D.destroy === 'function'
) {
window.EBAvatar3D.destroy();
}
} }
async function maybeResume() { async function maybeResume() {
@ -879,11 +1018,34 @@ async function loadChat() {
if (!config.voiceEnabled) { if (!config.voiceEnabled) {
throw new Error('Voice is not enabled for this chatbot'); throw new Error('Voice is not enabled for this chatbot');
} }
videoAvatarOn = Boolean(config.videoAvatarEnabled || config.avatarEnabled); videoAvatarOn = Boolean(
applyStageMode(); config.videoAvatarEnabled ||
config.avatarEnabled
);
if (videoAvatarOn) {
if (
!window.EBAvatar3D ||
typeof window.EBAvatar3D.init !== 'function'
) {
throw new Error(
'3D avatar client failed to load'
);
}
await window.EBAvatar3D.init({
container: avatarStageEl,
modelUrl: '/embed/avatar/model.glb',
});
console.info(
'[avatar3d] avatar initialized'
);
}
} }
async function startSession() { async function startSession() {
console.log('[DEBUG] startSession() START');
if (started) return; if (started) return;
started = true; started = true;
hideStartOverlay(); hideStartOverlay();
@ -898,6 +1060,11 @@ async function startSession() {
} }
await loadChat(); await loadChat();
console.log('[DEBUG] loadChat() DONE', {
videoAvatarOn,
avatarStageEl: !!avatarStageEl,
EBAvatar3D: !!window.EBAvatar3D
});
if (!navigator.mediaDevices?.getUserMedia) { if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('This browser cannot access the microphone'); throw new Error('This browser cannot access the microphone');
@ -912,7 +1079,7 @@ async function startSession() {
sessionId = granted.sessionId || sessionId; sessionId = granted.sessionId || sessionId;
sessionAlive = true; sessionAlive = true;
if (!videoAvatarOn) startWaveLoop(); // if (!videoAvatarOn) startWaveLoop();
const room = new Livekit.Room({ const room = new Livekit.Room({
adaptiveStream: false, adaptiveStream: false,
@ -932,7 +1099,13 @@ async function startSession() {
setStatus('Disconnected'); setStatus('Disconnected');
} }
}); });
room.on(Livekit.RoomEvent.TrackSubscribed, (track, _pub, participant) => { room.on(Livekit.RoomEvent.TrackSubscribed, (track, pub, participant) => {
console.log('[DEBUG] TrackSubscribed', {
kind: track?.kind,
source: pub?.source,
participant: participant?.identity,
trackSid: pub?.trackSid
});
handleIncomingAudio(track, participant, room); handleIncomingAudio(track, participant, room);
}); });
if (Livekit.RoomEvent.LocalTrackPublished) { if (Livekit.RoomEvent.LocalTrackPublished) {
@ -959,8 +1132,9 @@ async function startSession() {
const local = isLocalParticipant(room, participant); const local = isLocalParticipant(room, participant);
(segments || []).forEach((seg) => { (segments || []).forEach((seg) => {
if (!seg || !seg.final) return; if (!seg || !seg.final) return;
if (agentPipelineState === 'speaking' && local) return; if (agentPipelineState === 'speaking' && local) {
ensureSimliConnected(local ? 'user-speech' : 'agent-speech'); return;
}
if (rememberTranscript(seg.id || `${local ? 'user' : 'bot'}:${seg.text}`)) return; if (rememberTranscript(seg.id || `${local ? 'user' : 'bot'}:${seg.text}`)) return;
if (!local && pendingGreeting && normalizeSpeech(seg.text) === normalizeSpeech(pendingGreeting)) { if (!local && pendingGreeting && normalizeSpeech(seg.text) === normalizeSpeech(pendingGreeting)) {
showPendingGreeting(); showPendingGreeting();
@ -970,17 +1144,67 @@ async function startSession() {
}); });
}); });
room.on(Livekit.RoomEvent.ActiveSpeakersChanged, (speakers) => { room.on(Livekit.RoomEvent.ActiveSpeakersChanged, (speakers) => {
if ((speakers || []).length) ensureSimliConnected('speakers'); if (videoAvatarOn) return;
if (simliIdle || simliRestarting || agentPipelineState === 'speaking' || agentPipelineState === 'thinking') {
return; const localId = room.localParticipant?.identity;
}
if (videoAvatarOn) return; const agentSpeaking = (speakers || []).some(
const localId = room.localParticipant?.identity; (p) => p?.identity && p.identity !== localId
const agentSpeaking = (speakers || []).some((p) => p?.identity && p.identity !== localId); );
setStatus(agentSpeaking ? 'Speaking' : 'Listening');
setStatus(agentSpeaking ? 'Speaking' : 'Listening');
});
room.on(Livekit.RoomEvent.ParticipantConnected, (participant) => {
console.log('[DEBUG] ParticipantConnected', {
identity: participant?.identity,
sid: participant?.sid,
}); });
participant.trackPublications.forEach((pub) => {
console.log('[DEBUG] Remote publication', {
kind: pub?.kind,
source: pub?.source,
trackSid: pub?.trackSid,
subscribed: pub?.isSubscribed,
hasTrack: !!pub?.track,
});
if (pub?.track) {
handleIncomingAudio(pub.track, participant, room);
}
});
});
room.on(Livekit.RoomEvent.ParticipantConnected, (participant) => {
console.log('[DEBUG] ParticipantConnected', {
identity: participant?.identity,
sid: participant?.sid,
});
});
room.on(Livekit.RoomEvent.ParticipantDisconnected, (participant) => {
console.log('[DEBUG] ParticipantDisconnected', {
identity: participant?.identity,
sid: participant?.sid,
});
});
room.on(Livekit.RoomEvent.LocalTrackPublished, (publication) => {
console.log('[DEBUG] LocalTrackPublished', {
kind: publication?.kind,
source: publication?.source,
trackSid: publication?.trackSid,
});
});
await room.connect(granted.url, granted.token); await room.connect(granted.url, granted.token);
console.log('[DEBUG] LiveKit CONNECTED', {
localParticipant: room.localParticipant?.identity,
remoteParticipants: room.remoteParticipants?.size
});
if (typeof room.startAudio === 'function') { if (typeof room.startAudio === 'function') {
await room.startAudio().catch(() => {}); await room.startAudio().catch(() => {});
} }
@ -1004,6 +1228,11 @@ async function startSession() {
} }
if (!micPub) throw new Error('Microphone did not publish. Allow mic access and try again.'); if (!micPub) throw new Error('Microphone did not publish. Allow mic access and try again.');
micPublished = true; micPublished = true;
console.log('[DEBUG] MIC PUBLISHED', {
trackSid: micPub?.trackSid,
kind: micPub?.kind,
source: micPub?.source,
});
micListening = true; micListening = true;
const localMedia = mediaTrackFrom(micPub.track); const localMedia = mediaTrackFrom(micPub.track);
if (localMedia) hookWaveAnalyser(localMedia, 'user'); if (localMedia) hookWaveAnalyser(localMedia, 'user');
@ -1018,17 +1247,25 @@ async function startSession() {
setStatus('Listening'); setStatus('Listening');
return; return;
} }
setStatus('Connecting avatar…'); // setStatus('Connecting avatar…');
try { // try {
await Promise.race([ // await Promise.race([
startSimliOverlay(), // startSimliOverlay(),
new Promise((resolve) => setTimeout(resolve, 2500)), // new Promise((resolve) => setTimeout(resolve, 2500)),
]); // ]);
} catch (err) { // } catch (err) {
console.warn('[avatar] simli overlay', err); // console.warn('[avatar] simli overlay', err);
} // }
await signalAvatarReady(room); // await signalAvatarReady(room);
applyAgentState(readRemoteAgentState(room) || agentPipelineState); // applyAgentState(readRemoteAgentState(room) || agentPipelineState);
setStatus('Listening');
await signalAvatarReady(room);
applyAgentState(
readRemoteAgentState(room) ||
agentPipelineState
);
} }
function needsGesture(err) { function needsGesture(err) {

646
public/embed/avatar/avatar3d.js Normal file
View File

@ -0,0 +1,646 @@
(function () {
'use strict';
let THREE = null;
let GLTFLoader = null;
let scene = null;
let camera = null;
let renderer = null;
let model = null;
let container = null;
let animationFrame = 0;
let analyser = null;
let frequencyData = null;
const morphs = {
jawOpen: [],
mouthClose: [],
sil: [],
};
let initialized = false;
// --------------------------------------------------
// Load Three.js + GLTFLoader
// --------------------------------------------------
async function loadThree() {
if (THREE && GLTFLoader) {
return;
}
const threeModule = await import(
'https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js'
);
THREE = threeModule;
const loaderModule = await import(
'https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/loaders/GLTFLoader.js'
);
GLTFLoader = loaderModule.GLTFLoader;
}
// --------------------------------------------------
// Initialize
// --------------------------------------------------
async function init(options = {}) {
if (initialized) {
return window.EBAvatar3D;
}
container =
options.container ||
document.getElementById('eba-avatar-stage');
if (!container) {
throw new Error(
'[avatar3d] #eba-avatar-stage not found'
);
}
await loadThree();
createRenderer();
createScene();
createCamera();
createLights();
await loadModel(
options.modelUrl || '/embed/avatar/model.glb'
);
setupResize();
initialized = true;
startRenderLoop();
console.info('[avatar3d] initialized');
return window.EBAvatar3D;
}
// --------------------------------------------------
// Renderer
// --------------------------------------------------
function createRenderer() {
renderer = new THREE.WebGLRenderer({
antialias: true,
alpha: false,
powerPreference: 'high-performance',
});
renderer.setPixelRatio(
Math.min(window.devicePixelRatio || 1, 2)
);
renderer.setSize(
container.clientWidth || 640,
container.clientHeight || 640,
false
);
renderer.outputColorSpace = THREE.SRGBColorSpace;
renderer.domElement.id = 'eba-avatar-canvas';
renderer.domElement.style.display = 'block';
renderer.domElement.style.width = '100%';
renderer.domElement.style.height = '100%';
container.appendChild(renderer.domElement);
}
// --------------------------------------------------
// Scene
// --------------------------------------------------
function createScene() {
scene = new THREE.Scene();
scene.background = new THREE.Color(0x020617);
}
// --------------------------------------------------
// Camera
// --------------------------------------------------
function createCamera() {
const width =
container.clientWidth || 640;
const height =
container.clientHeight || 640;
camera = new THREE.PerspectiveCamera(
30,
width / height,
0.01,
100
);
camera.position.set(
0,
1.55,
2.7
);
camera.lookAt(
0,
1.45,
0
);
}
// --------------------------------------------------
// Lights
// --------------------------------------------------
function createLights() {
const ambient =
new THREE.HemisphereLight(
0xffffff,
0x223344,
2
);
scene.add(ambient);
const key =
new THREE.DirectionalLight(
0xffffff,
2.5
);
key.position.set(2, 3, 4);
scene.add(key);
const fill =
new THREE.DirectionalLight(
0xaac8ff,
1
);
fill.position.set(-3, 2, 2);
scene.add(fill);
}
// --------------------------------------------------
// Load GLB
// --------------------------------------------------
async function loadModel(url) {
return new Promise((resolve, reject) => {
const loader = new GLTFLoader();
console.info(
'[avatar3d] loading model:',
url
);
loader.load(
url,
(gltf) => {
model = gltf.scene;
scene.add(model);
inspectModel(model);
fitModel();
console.info(
'[avatar3d] model loaded'
);
resolve(model);
},
(progress) => {
if (progress.total) {
const percent =
(progress.loaded /
progress.total) *
100;
console.info(
`[avatar3d] loading ${percent.toFixed(0)}%`
);
}
},
(error) => {
console.error(
'[avatar3d] model loading failed',
error
);
reject(error);
}
);
});
}
// --------------------------------------------------
// Find morph targets
// --------------------------------------------------
function inspectModel(root) {
Object.keys(morphs).forEach((key) => {
morphs[key].length = 0;
});
root.traverse((object) => {
if (!object.isMesh) {
return;
}
if (
!object.morphTargetDictionary ||
!object.morphTargetInfluences
) {
return;
}
console.info(
'[avatar3d] morph mesh:',
object.name
);
registerMorph(
object,
'jawOpen'
);
registerMorph(
object,
'mouthClose'
);
registerMorph(
object,
'sil'
);
});
console.info(
'[avatar3d] jawOpen targets:',
morphs.jawOpen.length
);
}
function registerMorph(mesh, name) {
const dictionary =
mesh.morphTargetDictionary;
if (
dictionary &&
Object.prototype.hasOwnProperty.call(
dictionary,
name
)
) {
morphs[name].push({
mesh,
index: dictionary[name],
});
}
}
// --------------------------------------------------
// Fit model
// --------------------------------------------------
function fitModel() {
if (!model) {
return;
}
const box =
new THREE.Box3().setFromObject(model);
const size =
new THREE.Vector3();
const center =
new THREE.Vector3();
box.getSize(size);
box.getCenter(center);
console.info(
'[avatar3d] model size:',
size
);
model.position.x -= center.x;
model.position.y -= box.min.y;
model.position.z -= center.z;
const maxDimension =
Math.max(
size.x,
size.y,
size.z
);
if (maxDimension > 5) {
const scale =
2 / maxDimension;
model.scale.setScalar(scale);
}
}
// --------------------------------------------------
// Audio analyser
// --------------------------------------------------
function setAudioAnalyser(nextAnalyser) {
analyser = nextAnalyser;
if (analyser) {
frequencyData =
new Uint8Array(
analyser.frequencyBinCount
);
console.info(
'[avatar3d] audio analyser connected'
);
} else {
frequencyData = null;
console.info(
'[avatar3d] audio analyser disconnected'
);
}
}
function getAudioLevel() {
if (!analyser || !frequencyData) {
return 0;
}
analyser.getByteFrequencyData(
frequencyData
);
let sum = 0;
for (
let i = 0;
i < frequencyData.length;
i++
) {
const value =
frequencyData[i] / 255;
sum += value * value;
}
const rms =
Math.sqrt(
sum /
frequencyData.length
);
return Math.min(
1,
rms * 2.5
);
}
// --------------------------------------------------
// Lip movement
// --------------------------------------------------
function updateMouth() {
const level =
getAudioLevel();
const target =
Math.min(
1,
Math.max(
0,
level * 2.2
)
);
// Jaw opens when agent speaks.
morphs.jawOpen.forEach(
(targetInfo) => {
const current =
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] || 0;
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] =
THREE.MathUtils.lerp(
current,
target,
0.35
);
}
);
// Mouth close when silent.
morphs.mouthClose.forEach(
(targetInfo) => {
const current =
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] || 0;
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] =
THREE.MathUtils.lerp(
current,
Math.max(
0,
1 - target
),
0.15
);
}
);
// Silence target.
morphs.sil.forEach(
(targetInfo) => {
const current =
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] || 0;
const silence =
target < 0.03
? 1
: 0;
targetInfo.mesh
.morphTargetInfluences[
targetInfo.index
] =
THREE.MathUtils.lerp(
current,
silence,
0.15
);
}
);
}
// --------------------------------------------------
// Render loop
// --------------------------------------------------
function startRenderLoop() {
if (animationFrame) {
return;
}
const render = () => {
animationFrame =
requestAnimationFrame(
render
);
updateMouth();
if (
renderer &&
scene &&
camera
) {
renderer.render(
scene,
camera
);
}
};
render();
}
// --------------------------------------------------
// Resize
// --------------------------------------------------
function setupResize() {
window.addEventListener(
'resize',
resize,
{ passive: true }
);
resize();
}
function resize() {
if (
!renderer ||
!camera ||
!container
) {
return;
}
const width =
container.clientWidth || 640;
const height =
container.clientHeight || 640;
camera.aspect =
width / height;
camera.updateProjectionMatrix();
renderer.setSize(
width,
height,
false
);
}
// --------------------------------------------------
// Destroy
// --------------------------------------------------
function destroy() {
if (animationFrame) {
cancelAnimationFrame(
animationFrame
);
animationFrame = 0;
}
if (renderer) {
renderer.dispose();
if (
renderer.domElement &&
renderer.domElement.parentNode
) {
renderer.domElement.parentNode.removeChild(
renderer.domElement
);
}
}
scene = null;
camera = null;
renderer = null;
model = null;
analyser = null;
frequencyData = null;
Object.keys(morphs).forEach(
(key) => {
morphs[key].length = 0;
}
);
initialized = false;
}
// --------------------------------------------------
// Public API
// --------------------------------------------------
window.EBAvatar3D = {
init,
setAudioAnalyser,
destroy,
get initialized() {
return initialized;
},
get model() {
return model;
},
};
})();

BIN
public/embed/avatar/model.glb Normal file
View File

Binary file not shown.

98
public/embed/simli-client.iife.js Normal file
View File

File diff suppressed because one or more lines are too long

24
src/app.js
View File

@ -159,6 +159,30 @@ const createServer = async () => {
options: { auth: false }, options: { auth: false },
handler: async (request, h) => sendFile(h, 'embed-avatar.html', 'text/html'), handler: async (request, h) => sendFile(h, 'embed-avatar.html', 'text/html'),
}, },
{
method: 'GET',
path: '/embed/avatar/avatar3d.js',
options: { auth: false },
handler: async (request, h) =>
sendFile(
h,
'embed/avatar/avatar3d.js',
'application/javascript'
),
},
{
method: 'GET',
path: '/embed/avatar/model.glb',
options: { auth: false },
handler: async (request, h) =>
sendFile(
h,
'embed/avatar/model.glb',
'model/gltf-binary',
null
),
},
{ {
method: 'GET', method: 'GET',
path: '/embed/avatar-page.js', path: '/embed/avatar-page.js',

28
src/services/whatsappReception.service.js
View File

@ -8,11 +8,13 @@ const {
membershipAllowed, membershipAllowed,
} = require('../utils/participantMatch'); } = require('../utils/participantMatch');
const { getOrgWhatsAppCredentials } = require('../utils/whatsappCloud'); const { getOrgWhatsAppCredentials } = require('../utils/whatsappCloud');
const { sendTextMessage, downloadWhatsAppMedia } = require('../utils/whatsappSend'); const { sendTextMessage, sendMessageWithCtas, downloadWhatsAppMedia } = require('../utils/whatsappSend');
const { decodeParticipantIdFromImage } = require('../utils/badgeQr'); const { decodeParticipantIdFromImage } = require('../utils/badgeQr');
const { detectBadgeTextLines } = require('../utils/badgeOcr'); const { detectBadgeTextLines } = require('../utils/badgeOcr');
const { const {
interpolateReception, interpolateReception,
interpolateCtas,
buildEngagementUrl,
defaultBadgePrompt, defaultBadgePrompt,
defaultStaffHandoff, defaultStaffHandoff,
defaultWelcome, defaultWelcome,
@ -108,7 +110,7 @@ async function credsForBot(bot) {
return getOrgWhatsAppCredentials(String(bot.organization)); return getOrgWhatsAppCredentials(String(bot.organization));
} }
async function sendToSession(bot, session, text) { async function sendToSession(bot, session, text, ctas = []) {
const creds = await credsForBot(bot); const creds = await credsForBot(bot);
if (!creds?.accessToken || !creds.phoneNumberId) { if (!creds?.accessToken || !creds.phoneNumberId) {
logWhatsApp('send.skip', { logWhatsApp('send.skip', {
@ -120,11 +122,18 @@ async function sendToSession(bot, session, text) {
return; return;
} }
try { try {
logWhatsApp('send.text', { logWhatsApp(ctas.length ? 'send.interactive' : 'send.text', {
to: maskPhone(session.phoneNumber), to: maskPhone(session.phoneNumber),
preview: String(text || '').slice(0, 80), preview: String(text || '').slice(0, 80),
}); });
await sendTextMessage(creds.phoneNumberId, creds.accessToken, session.phoneNumber, text); if (ctas.length) {
await sendMessageWithCtas(creds.phoneNumberId, creds.accessToken, session.phoneNumber, {
body: text,
ctas,
});
} else {
await sendTextMessage(creds.phoneNumberId, creds.accessToken, session.phoneNumber, text);
}
} catch (err) { } catch (err) {
console.error('[whatsapp] reception send failed', err.message); console.error('[whatsapp] reception send failed', err.message);
} }
@ -151,7 +160,16 @@ async function sendWelcomeAndDrips(bot, session, participant, eventName) {
eventId: session.eventId || '', eventId: session.eventId || '',
participantId: participant?._id ? String(participant._id) : '', participantId: participant?._id ? String(participant._id) : '',
}); });
await sendToSession(bot, session, text); const engagementUrl = buildEngagementUrl(session.eventId || '', participant?._id ? String(participant._id) : '');
const ctas = interpolateCtas(bot.reception?.welcomeCtas, {
firstName,
profileName,
eventName,
eventId: session.eventId || '',
participantId: participant?._id ? String(participant._id) : '',
engagementUrl,
});
await sendToSession(bot, session, text, ctas);
session.welcomeSentAt = new Date(); session.welcomeSentAt = new Date();
logWhatsApp('welcome.queued', { participantId: String(participant._id), eventName }); logWhatsApp('welcome.queued', { participantId: String(participant._id), eventName });
await session.save(); await session.save();

1
src/utils/receptionCopy.js
View File

@ -33,6 +33,7 @@ function applyReceptionPlaceholders(text, vars) {
.replace(/\{eventName\}/gi, event) .replace(/\{eventName\}/gi, event)
.replace(/\{eventId\}/gi, eid) .replace(/\{eventId\}/gi, eid)
.replace(/\{participantId\}/gi, pid) .replace(/\{participantId\}/gi, pid)
.replace(/\{url\}/gi, url)
.replace(/\{engagementUrl\}/gi, url); .replace(/\{engagementUrl\}/gi, url);
} }

69
src/utils/simliConcierge.js Normal file
View File

@ -0,0 +1,69 @@
/**
* Simli AI avatar session tokens.
* The browser must not hold the Simli API key — it calls our endpoint, which
* exchanges {faceId, ...} for a session_token via api.simli.ai/compose/token.
*/
const SIMLI_API_BASE = 'https://api.simli.ai';
function getSimliCredentials() {
const apiKey = String(process.env.SIMLI_API_KEY || '').trim();
const faceId = String(process.env.SIMLI_FACE_ID || '').trim();
return { apiKey, faceId, configured: Boolean(apiKey && faceId) };
}
/**
* POST https://api.simli.ai/compose/token
* body: { faceId, handleSilence, maxSessionLength, maxIdleTime, model? }
* header: x-simli-api-key
* -> { session_token }
*/
async function generateSimliSessionToken() {
const { apiKey, faceId } = getSimliCredentials();
if (!apiKey || !faceId) throw new Error('Simli is not configured');
const res = await fetch(`${SIMLI_API_BASE}/compose/token`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'x-simli-api-key': apiKey },
body: JSON.stringify({
faceId,
handleSilence: true,
maxSessionLength: 900,
maxIdleTime: 300,
}),
});
const text = await res.text();
if (!res.ok) {
throw new Error(`Simli token request failed (${res.status}): ${text.slice(0, 200)}`);
}
let json;
try {
json = JSON.parse(text);
} catch {
throw new Error(`Simli token response not JSON: ${text.slice(0, 200)}`);
}
if (!json.session_token) throw new Error('Simli token response missing session_token');
return json.session_token;
}
/**
* GET https://api.simli.ai/compose/ice
* header: x-simli-api-key
* -> RTCIceServer[] (falls back to public STUN)
*/
async function generateIceServers() {
const { apiKey } = getSimliCredentials();
try {
const res = await fetch(`${SIMLI_API_BASE}/compose/ice`, {
method: 'GET',
headers: { 'Content-Type': 'application/json', 'x-simli-api-key': apiKey },
});
if (!res.ok) throw new Error(`status ${res.status}`);
const ice = await res.json();
if (!Array.isArray(ice) || ice.length === 0) throw new Error('empty ICE list');
return ice;
} catch {
return [{ urls: ['stun:stun.l.google.com:19302'] }];
}
}
module.exports = { getSimliCredentials, generateSimliSessionToken, generateIceServers };

4
src/utils/whatsappCtas.js
View File

@ -7,7 +7,7 @@
* @returns {{ type: 'reply'|'url', label: string, value: string }[]} * @returns {{ type: 'reply'|'url', label: string, value: string }[]}
*/ */
function isPlaceholderCtaUrl(value) { function isPlaceholderCtaUrl(value) {
return /\{engagementUrl\}|\{eventId\}|\{participantId\}/i.test(String(value || '')); return /\{url\}|\{engagementUrl\}|\{eventId\}|\{participantId\}/i.test(String(value || ''));
} }
function isHttpCtaUrl(value) { function isHttpCtaUrl(value) {
@ -43,7 +43,7 @@ function normalizeWhatsAppCtas(raw, opts = {}) {
if (type === 'url') { if (type === 'url') {
if (!isHttpCtaUrl(value) && !isPlaceholderCtaUrl(value)) { if (!isHttpCtaUrl(value) && !isPlaceholderCtaUrl(value)) {
if (strict) { if (strict) {
throw new Error('URL buttons must start with http:// or https://, or use {engagementUrl}'); throw new Error('URL buttons must start with http:// or https://, or use {Url}');
} }
continue; continue;
} }