From 29acdefafaf0195edd00c767492abcaa5754a43c Mon Sep 17 00:00:00 2001 From: Ravi Gulhane <> Date: Mon, 21 Sep 2026 10:52:06 +0530 Subject: [PATCH] Changes --- README.md | 25 +- ecosystem.config.cjs | 20 +- eventbuzz-concierge-implementation-plan.md | 263 +++ package-lock.json | 118 +- package.json | 8 +- public/embed-avatar.html | 47 + public/embed/avatar-page.css | 299 +++ public/embed/avatar-page.js | 1071 ++++++++++ public/embed/concierge-widget.css | 4 + public/embed/concierge-widget.js | 31 +- public/embed/simli-client.js | 66 + src/agent/cardsFromTools.js | 115 ++ src/agent/compact.js | 37 + src/agent/eventBrief.js | 180 ++ src/agent/groundReply.js | 75 + src/agent/prompts.js | 64 + src/agent/runner.js | 263 +++ src/agent/tools/actions.tools.js | 70 + src/agent/tools/index.js | 139 ++ src/agent/tools/onsite.tools.js | 113 ++ src/agent/tools/personal.tools.js | 107 + src/agent/tools/programme.tools.js | 85 + src/app.js | 26 +- src/controllers/publicConcierge.controller.js | 46 +- src/prompts/call2/agenda/agenda_by_speaker.js | 9 - src/prompts/call2/agenda/agenda_by_track.js | 10 - .../call2/agenda/agenda_conflict_check.js | 10 - .../call2/agenda/agenda_current_now.js | 10 - .../call2/agenda/agenda_download_materials.js | 10 - .../call2/agenda/agenda_personal_schedule.js | 11 - .../call2/agenda/agenda_session_details.js | 15 - .../call2/agenda/agenda_upcoming_next.js | 10 - src/prompts/call2/agenda/index.js | 43 - src/prompts/call2/conversation.js | 17 - .../call2/exhibitor/exhibitor_book_meeting.js | 9 - .../call2/exhibitor/exhibitor_by_category.js | 10 - .../exhibitor/exhibitor_demo_schedule.js | 10 - .../call2/exhibitor/exhibitor_location.js | 11 - src/prompts/call2/exhibitor/index.js | 34 - src/prompts/call2/exhibitor/main.js | 19 - src/prompts/call2/local/accommodation.js | 13 - src/prompts/call2/local/index.js | 31 - src/prompts/call2/local/main.js | 18 - src/prompts/call2/local/tourist_places.js | 12 - src/prompts/call2/main.js | 24 - src/prompts/call2/speaker/index.js | 32 - src/prompts/call2/speaker/main.js | 19 - .../call2/speaker/speaker_bio_lookup.js | 13 - .../call2/speaker/speaker_contact_info.js | 12 - .../call2/speaker/speaker_meet_and_greet.js | 11 - src/prompts/call2/system/chitchat_greeting.js | 8 - src/prompts/call2/system/emergency_medical.js | 10 - src/prompts/call2/system/event_adjacent.js | 18 - .../call2/system/human_agent_escalation.js | 11 - src/prompts/call2/system/index.js | 30 - src/prompts/call2/system/lost_and_found.js | 8 - src/prompts/call2/system/main.js | 19 - src/prompts/call2/system/out_of_scope.js | 8 - .../call2/tech/app_feature_guidance.js | 8 - src/prompts/call2/tech/app_issue_account.js | 9 - src/prompts/call2/tech/app_issue_streaming.js | 9 - src/prompts/call2/tech/index.js | 26 - src/prompts/call2/tech/main.js | 17 - src/prompts/call2/tech/wifi_credentials.js | 8 - src/prompts/call2/ticket/index.js | 39 - src/prompts/call2/ticket/main.js | 17 - .../call2/ticket/ticket_document_receipt.js | 11 - src/prompts/call2/ticket/ticket_email_qr.js | 12 - .../call2/ticket/ticket_how_to_register.js | 12 - src/prompts/call2/ticket/ticket_qr_display.js | 10 - .../call2/ticket/ticket_status_lookup.js | 10 - .../call2/ticket/ticket_transfer_request.js | 11 - src/prompts/call2/ticket/ticket_upgrade.js | 11 - src/prompts/call2/venue/index.js | 41 - src/prompts/call2/venue/main.js | 20 - .../call2/venue/venue_accessibility.js | 10 - .../call2/venue/venue_food_beverage.js | 12 - src/prompts/call2/venue/venue_map_request.js | 10 - .../call2/venue/venue_nearest_amenity.js | 10 - .../call2/venue/venue_parking_transport.js | 18 - .../call2/venue/venue_travel_directions.js | 20 - src/prompts/call2/venue/venue_wayfinding.js | 10 - src/prompts/step1.js | 127 -- src/rag/retriever.js | 80 + src/routes/publicConcierge.routes.js | 11 + src/services/identity.service.js | 5 +- src/services/orchestrator.service.js | 432 +--- src/services/retrieveForTurn.js | 301 --- src/services/step1.service.js | 415 ---- src/services/step2.service.js | 1775 ----------------- src/services/ticketData.service.js | 6 +- src/services/venueData.service.js | 17 +- src/services/whatsappDrip.service.js | 71 +- src/services/whatsappFollowup.service.js | 18 +- src/services/whatsappInbound.service.js | 11 + src/services/whatsappReception.service.js | 2 + src/utils/bedrock.js | 168 +- src/utils/call2Context.js | 730 ------- src/utils/domainAction.js | 334 ---- src/utils/receptionCopy.js | 55 +- src/utils/registerIntent.js | 46 + src/utils/simli.js | 51 + src/utils/webChatTurn.js | 4 +- src/utils/whatsappCtas.js | 52 +- src/utils/whatsappFormat.js | 8 +- src/utils/whatsappSend.js | 41 +- src/workers/livekitVoiceAgent.js | 96 +- 107 files changed, 3990 insertions(+), 5034 deletions(-) create mode 100644 eventbuzz-concierge-implementation-plan.md create mode 100644 public/embed-avatar.html create mode 100644 public/embed/avatar-page.css create mode 100644 public/embed/avatar-page.js create mode 100644 public/embed/simli-client.js create mode 100644 src/agent/cardsFromTools.js create mode 100644 src/agent/compact.js create mode 100644 src/agent/eventBrief.js create mode 100644 src/agent/groundReply.js create mode 100644 src/agent/prompts.js create mode 100644 src/agent/runner.js create mode 100644 src/agent/tools/actions.tools.js create mode 100644 src/agent/tools/index.js create mode 100644 src/agent/tools/onsite.tools.js create mode 100644 src/agent/tools/personal.tools.js create mode 100644 src/agent/tools/programme.tools.js delete mode 100644 src/prompts/call2/agenda/agenda_by_speaker.js delete mode 100644 src/prompts/call2/agenda/agenda_by_track.js delete mode 100644 src/prompts/call2/agenda/agenda_conflict_check.js delete mode 100644 src/prompts/call2/agenda/agenda_current_now.js delete mode 100644 src/prompts/call2/agenda/agenda_download_materials.js delete mode 100644 src/prompts/call2/agenda/agenda_personal_schedule.js delete mode 100644 src/prompts/call2/agenda/agenda_session_details.js delete mode 100644 src/prompts/call2/agenda/agenda_upcoming_next.js delete mode 100644 src/prompts/call2/agenda/index.js delete mode 100644 src/prompts/call2/conversation.js delete mode 100644 src/prompts/call2/exhibitor/exhibitor_book_meeting.js delete mode 100644 src/prompts/call2/exhibitor/exhibitor_by_category.js delete mode 100644 src/prompts/call2/exhibitor/exhibitor_demo_schedule.js delete mode 100644 src/prompts/call2/exhibitor/exhibitor_location.js delete mode 100644 src/prompts/call2/exhibitor/index.js delete mode 100644 src/prompts/call2/exhibitor/main.js delete mode 100644 src/prompts/call2/local/accommodation.js delete mode 100644 src/prompts/call2/local/index.js delete mode 100644 src/prompts/call2/local/main.js delete mode 100644 src/prompts/call2/local/tourist_places.js delete mode 100644 src/prompts/call2/main.js delete mode 100644 src/prompts/call2/speaker/index.js delete mode 100644 src/prompts/call2/speaker/main.js delete mode 100644 src/prompts/call2/speaker/speaker_bio_lookup.js delete mode 100644 src/prompts/call2/speaker/speaker_contact_info.js delete mode 100644 src/prompts/call2/speaker/speaker_meet_and_greet.js delete mode 100644 src/prompts/call2/system/chitchat_greeting.js delete mode 100644 src/prompts/call2/system/emergency_medical.js delete mode 100644 src/prompts/call2/system/event_adjacent.js delete mode 100644 src/prompts/call2/system/human_agent_escalation.js delete mode 100644 src/prompts/call2/system/index.js delete mode 100644 src/prompts/call2/system/lost_and_found.js delete mode 100644 src/prompts/call2/system/main.js delete mode 100644 src/prompts/call2/system/out_of_scope.js delete mode 100644 src/prompts/call2/tech/app_feature_guidance.js delete mode 100644 src/prompts/call2/tech/app_issue_account.js delete mode 100644 src/prompts/call2/tech/app_issue_streaming.js delete mode 100644 src/prompts/call2/tech/index.js delete mode 100644 src/prompts/call2/tech/main.js delete mode 100644 src/prompts/call2/tech/wifi_credentials.js delete mode 100644 src/prompts/call2/ticket/index.js delete mode 100644 src/prompts/call2/ticket/main.js delete mode 100644 src/prompts/call2/ticket/ticket_document_receipt.js delete mode 100644 src/prompts/call2/ticket/ticket_email_qr.js delete mode 100644 src/prompts/call2/ticket/ticket_how_to_register.js delete mode 100644 src/prompts/call2/ticket/ticket_qr_display.js delete mode 100644 src/prompts/call2/ticket/ticket_status_lookup.js delete mode 100644 src/prompts/call2/ticket/ticket_transfer_request.js delete mode 100644 src/prompts/call2/ticket/ticket_upgrade.js delete mode 100644 src/prompts/call2/venue/index.js delete mode 100644 src/prompts/call2/venue/main.js delete mode 100644 src/prompts/call2/venue/venue_accessibility.js delete mode 100644 src/prompts/call2/venue/venue_food_beverage.js delete mode 100644 src/prompts/call2/venue/venue_map_request.js delete mode 100644 src/prompts/call2/venue/venue_nearest_amenity.js delete mode 100644 src/prompts/call2/venue/venue_parking_transport.js delete mode 100644 src/prompts/call2/venue/venue_travel_directions.js delete mode 100644 src/prompts/call2/venue/venue_wayfinding.js delete mode 100644 src/prompts/step1.js create mode 100644 src/rag/retriever.js delete mode 100644 src/services/retrieveForTurn.js delete mode 100644 src/services/step1.service.js delete mode 100644 src/services/step2.service.js delete mode 100644 src/utils/call2Context.js delete mode 100644 src/utils/domainAction.js create mode 100644 src/utils/registerIntent.js create mode 100644 src/utils/simli.js diff --git a/README.md b/README.md index fa212f1..d19580e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # EventBuzz Concierge -Independent Node.js service for the EventBuzz Virtual Concierge chatbot. Uses Amazon Bedrock for Call 1 (intent classification). Connects to the same MongoDB as the main EventBuzz backend. +Independent Node.js service for the EventBuzz Virtual Concierge chatbot. Uses Amazon Bedrock **tool calling** (router + synthesis). Connects to the same MongoDB as the main EventBuzz backend. ## Setup @@ -30,26 +30,31 @@ First time on the server: create `/var/www/concierge/.env` (copy from `.env.exam ## Embed demo 1. Start the server. -2. Open [http://localhost:3010/embed/demo](http://localhost:3010/embed/demo). -3. Set `botId` in the page script to a real `ConciergeBot` document `_id` from MongoDB. +2. Open [http://localhost:3010/embed/demo](http://localhost:3010/embed/demo) for the floating widget, or [http://localhost:3010/embed/avatar](http://localhost:3010/embed/avatar) for the SIMLI avatar + chat page. +3. Set `botId` in the page script to a real `ConciergeBot` document `_id` from MongoDB. The avatar page also needs `SIMLI_API_KEY` and `SIMLI_FACE_ID` in `.env`, plus LiveKit voice. ## API | Method | Path | Purpose | |--------|------|---------| | GET | `/api/public/concierge/{botId}/config` | Widget greeting / config | -| POST | `/api/public/concierge/{botId}/chat` | Send message (Call 1 classification) | +| POST | `/api/public/concierge/{botId}/chat` | Send a chat turn (tool-calling agent) | | GET | `/api/public/concierge/{botId}/session/{sessionRef}` | Resume session | +| POST | `/api/public/concierge/{botId}/voice/token` | LiveKit visitor token | +| POST | `/api/public/concierge/{botId}/simli/token` | SIMLI session token (API key stays on the server) | | GET | `/webhook/whatsapp` | Meta webhook verification | | POST | `/webhook/whatsapp` | Inbound WhatsApp (reception + follow-up) | -## Pipeline +## Agent -1. **Call 1** (Nova Micro) — intent + entities -2. **Call 2** (Claude 3 Haiku) — agenda replies with Sessions DB context -3. **Bookmark mutations** — `agenda_bookmark_add` / `agenda_bookmark_remove` write `SessionBookmark` records (no Call 2) -4. **Email OTP** — required for personal schedule, conflict check, and bookmark mutations; auth is stored on the web chat session +One Bedrock model (`BEDROCK_AGENT_MODEL_ID`, defaulting to Claude Haiku 4.5) chooses EventBuzz tools, the service executes them against Mongo (identity injected from the chat session), then the same model writes the visitor-facing reply. -Set `BEDROCK_CALL2_MODEL_ID`, `OTP_PEPPER`, and SMTP/`MAIL_*` vars in `.env` (see `.env.example`). Without SMTP, OTP codes are logged to the server console. +Default model: `global.anthropic.claude-haiku-4-5-20251001-v1:0` (global inference profile — required in `ap-south-1`). Prompt caching is on by default (`BEDROCK_PROMPT_CACHE=true`): tools, the system prompt, and the static EVENT brief are marked with `cachePoint` (1-hour TTL). Haiku 4.5 only records a cache hit after ~4096 tokens in that prefix; check `cacheReadInputTokens` / `cacheWriteInputTokens` in Bedrock logs. Claude 3 Haiku (20240307) cannot use `cachePoint`. + +Tools: `get_my_tickets`, `get_my_agenda`, `email_my_ticket`, `get_agenda`, `get_speakers`, `get_exhibitors`, `get_venue`, `get_local`, `get_tech_help`, `search_faqs`, `save_session`, `remove_saved_session`, `contact_organizer`. + +Personal tools and mutations require email OTP (`identity.service`). Widget, WhatsApp, and LiveKit voice all share this runner. + +Set `BEDROCK_AGENT_MODEL_ID` or `BEDROCK_CALL2_MODEL_ID`, `OTP_PEPPER`, and SMTP/`MAIL_*` vars in `.env` (see `.env.example`). Without SMTP, OTP codes are logged to the server console. Reception WhatsApp: point the Meta webhook at `GET/POST /webhook/whatsapp` on this service (verify token `META_WHATSAPP_WEBHOOK_VERIFY_TOKEN`). Configure copy and drips in the dashboard **WhatsApp Chatbots → Reception** tab. WABA credentials come from Organisation WhatsApp settings. diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs index 0c3a1f2..a0e5054 100644 --- a/ecosystem.config.cjs +++ b/ecosystem.config.cjs @@ -1,11 +1,14 @@ /** - * PM2 — EventBuzz Virtual Concierge API (same server as backend). + * PM2 — EventBuzz Virtual Concierge API + LiveKit voice worker (same server as backend). * * cd /var/www/concierge && pm2 start ecosystem.config.cjs * pm2 save * - * Requires concierge/.env on the server (MONGODB_URI, AWS/Bedrock, MAIL_*). + * Requires concierge/.env on the server (MONGODB_URI, AWS/Bedrock, MAIL_*, LIVEKIT_*). * Prefer HOST=0.0.0.0 (or nginx → 127.0.0.1:3010 with HOST=localhost). + * + * LiveKit voice: LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET. + * One worker instance only (it registers with LiveKit Cloud). */ module.exports = { apps: [ @@ -21,5 +24,18 @@ module.exports = { NODE_ENV: 'production', }, }, + { + name: 'livekit-voice-worker', + script: 'src/workers/livekitVoiceAgent.js', + args: 'start', + instances: 1, + exec_mode: 'fork', + autorestart: true, + max_restarts: 20, + restart_delay: 5000, + env: { + NODE_ENV: 'production', + }, + }, ], }; diff --git a/eventbuzz-concierge-implementation-plan.md b/eventbuzz-concierge-implementation-plan.md new file mode 100644 index 0000000..75f7cd0 --- /dev/null +++ b/eventbuzz-concierge-implementation-plan.md @@ -0,0 +1,263 @@ +# Specification: Eventbuzz Virtual Concierge (Text & LiveKit Voice) + +## 1. System Overview & Architecture + +Build a multi-modal (Text & Voice) AI concierge agent for the **Eventbuzz** platform. The system uses an **Agentic Tool-Calling pattern** where a single LLM call acts as both intent recognizer and response generator. + +```text + ┌──────────────┐ + │ Client / Web │ + │ (Text / App) │ + └──────┬───────┘ + │ (HTTP/WS with JWT) + ▼ +┌──────────────┐ ┌────────────────────────────────────────────────────────┐ +│ LiveKit │ │ Concierge Agent Gateway │ +│ Voice Agent ├─►│ 1. Extract & Verify Auth (User ID, Event ID) │ +│ (SIP/WebRTC) │ │ 2. Construct LLM Context & Bind User-Scoped Tools │ +└──────────────┘ └──────────────────────────┬─────────────────────────────┘ + │ + ▼ + ┌────────────────────┐ + │ LLM (Tool Calling) │ + └──────────┬─────────┘ + │ + ┌───────────────────────┴───────────────────────┐ + ▼ ▼ + [Personal Info Intent] [Event Knowledge Intent] + Tool: `get_my_registration` Tool: `search_event_faq` + Backend API / DB query Vector DB / Hybrid Search + (Hard-scoped to Auth Context) (Scoped to Event ID) + │ │ + └───────────────────────┬───────────────────────┘ + ▼ + ┌───────────────────────┐ + │ LLM Synthesis Call │ + └───────────┬───────────┘ + ▼ + Formatted Text / Audio Response +``` + +--- + +## 2. Core Project Structure + +```text +src/ +├── agent/ +│ ├── prompts.ts # System instructions and tone configuration +│ ├── runner.ts # Core LLM chat and tool execution loop +│ └── tools/ +│ ├── index.ts # Tool registry and dynamic context binding +│ ├── personal.tools.ts # Ticket, registration, and user-specific queries +│ └── knowledge.tools.ts # RAG / FAQ retrieval tools +├── auth/ +│ └── context.ts # Session extraction (userId, eventId, role) +├── livekit/ +│ └── voice_worker.ts # LiveKit Agents SDK worker integration +├── rag/ +│ └── retriever.ts # Vector search or semantic retrieval service +└── server.ts # WebSocket / HTTP entry point for text chat +``` + +--- + +## 3. Tool Declarations & Context Binding + +To prevent privilege escalation or data leakage, **the LLM must never supply the `userId` as an argument**. The execution layer injects user identity from the authenticated session. + +### Tool Schemas (Provided to LLM) + +```typescript +export const conciergeTools = [ + { + name: "get_my_registration_and_tickets", + description: "Retrieves the current attendee's ticket tier, QR code data, payment status, and registered sessions. Requires no input arguments.", + parameters: { + type: "object", + properties: {}, + required: [] + } + }, + { + name: "search_event_info", + description: "Searches event details such as venue policies, schedules, speaker bios, dress code, Wi-Fi details, and general FAQs.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "Specific search term or question regarding the event." + } + }, + required: ["query"] + } + }, + { + name: "get_agenda_schedule", + description: "Fetches schedule items for a specific date, stage, or track.", + parameters: { + type: "object", + properties: { + date: { type: "string", description: "Date in YYYY-MM-DD format (optional)" }, + track: { type: "string", description: "Track or stage name (optional)" } + }, + required: [] + } + } +]; +``` + +### Context-Bound Tool Handlers + +```typescript +export interface SessionContext { + userId: string; + eventId: string; +} + +export function createToolExecutors(ctx: SessionContext) { + return { + get_my_registration_and_tickets: async () => { + // Injects ctx.userId automatically; LLM cannot spoof another user + return await db.registrations.findFirst({ + where: { userId: ctx.userId, eventId: ctx.eventId }, + include: { tickets: true, assignedSessions: true } + }); + }, + + search_event_info: async ({ query }: { query: string }) => { + return await vectorStore.similaritySearch(query, { + k: 4, + filter: { eventId: ctx.eventId } + }); + }, + + get_agenda_schedule: async (filters: { date?: string; track?: string }) => { + return await db.sessions.findMany({ + where: { + eventId: ctx.eventId, + ...(filters.date && { date: filters.date }), + ...(filters.track && { track: filters.track }) + } + }); + } + }; +} +``` + +--- + +## 4. System Prompt Design + +```text +You are the Eventbuzz Virtual Concierge for the event identified as {{eventName}}. +Your role is to assist participants with both logistical event questions and personal ticketing/registration details. + +Tone & Demeanor: +- Welcoming, clear, concise, and professional. +- For voice delivery (via LiveKit), keep responses brief (under 3 sentences when possible) and avoid formatting characters like markdown tables, bullets, or raw URLs. + +Tool Rules: +1. Whenever the user asks about their own status, tickets, check-in QR, or seat, invoke `get_my_registration_and_tickets`. +2. When the user asks general event questions (parking, dress code, speaker line-ups), invoke `search_event_info`. +3. NEVER assume personal registration details or make up ticket numbers. If a tool returns no record, inform the participant politely and suggest visiting the physical Help Desk. +``` + +--- + +## 5. Execution Loop (Text & Voice) + +```typescript +export async function runConciergeTurn( + messages: ChatMessage[], + ctx: SessionContext +) { + const toolExecutors = createToolExecutors(ctx); + + // First call: LLM determines whether to respond directly or invoke a tool + const response = await llm.chat({ + messages, + tools: conciergeTools, + toolChoice: "auto" + }); + + const toolCalls = response.message.tool_calls; + + // Case 1: Direct response (chit-chat or clarifications) + if (!toolCalls || toolCalls.length === 0) { + return response.message.content; + } + + // Case 2: Intent matched to tool + const toolResults = await Promise.all( + toolCalls.map(async (call) => { + const fn = toolExecutors[call.function.name]; + if (!fn) throw new Error(`Unknown tool: ${call.function.name}`); + + const args = JSON.parse(call.function.arguments || "{}"); + const result = await fn(args); + + return { + tool_call_id: call.id, + role: "tool", + name: call.function.name, + content: JSON.stringify(result) + }; + }) + ); + + // Synthesis call: Feed tool outputs back to LLM for final natural response + const finalResponse = await llm.chat({ + messages: [ + ...messages, + response.message, + ...toolResults + ] + }); + + return finalResponse.message.content; +} +``` + +--- + +## 6. LiveKit Voice Integration Spec + +Connect the agent to LiveKit rooms using the LiveKit Agents framework: + +1. **Room Entry & Token Validation:** When dispatching the voice worker to a participant's room, pass `userId` and `eventId` inside the LiveKit participant metadata or worker job metadata. +2. **STT & TTS Pipeline:** + * **STT:** Deepgram / Whisper stream. + * **LLM Engine:** Hook `runConciergeTurn` directly into the agent callback. + * **TTS:** Cartesia / ElevenLabs stream with low-latency configuration. +3. **Interrupt Handling:** Ensure LiveKit's turn-detection/VAD (Voice Activity Detection) cancels inflight LLM synthesis if the participant interrupts mid-sentence. + +--- + +## 7. Step-by-Step Implementation Tasks for Cursor + +```text +- [ ] Task 1: Setup data layer for event knowledge and registrations + - Create mock/read queries for `registrations`, `tickets`, and `sessions`. + - Implement basic RAG retriever filtered by `eventId`. + +- [ ] Task 2: Implement Tool Definitions and Executor Mapping + - Define JSON schemas in `src/agent/tools/`. + - Implement context-injection factory in `src/agent/tools/index.ts`. + +- [ ] Task 3: Build the Agent Orchestrator + - Implement `runConciergeTurn` in `src/agent/runner.ts`. + - Add conversational state management (in-memory or Redis session store). + +- [ ] Task 4: Expose Text Chat API + - Create a POST `/api/concierge/chat` route with JWT validation. + - Test via curl: + 1. General query ("Where is the keynote?") + 2. Personal query ("Do I have a VIP pass?") + +- [ ] Task 5: Implement LiveKit Voice Worker + - Configure `livekit-agents` worker in `src/livekit/voice_worker.ts`. + - Bind participant metadata to `SessionContext`. + - Ensure tool outputs format cleanly for speech (no bullet points or markdown tables). +``` diff --git a/package-lock.json b/package-lock.json index 3f9140a..9264f5a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,8 @@ "livekit-server-sdk": "^2.18.0", "mongoose": "^8.0.0", "nodemailer": "^6.9.7", - "qrcode": "^1.5.4" + "qrcode": "^1.5.4", + "simli-client": "^3.0.2" }, "engines": { "node": ">=20" @@ -2796,6 +2797,13 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "license": "MIT" }, + "node_modules/@types/dom-mediacapture-record": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.22.tgz", + "integrity": "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==", + "license": "MIT", + "peer": true + }, "node_modules/@types/node": { "version": "16.9.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", @@ -4025,6 +4033,36 @@ "node": ">=12.0.0" } }, + "node_modules/livekit-client": { + "version": "2.22.3", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.22.3.tgz", + "integrity": "sha512-jw9zBKXY5Gtr5MZ7vEON3QhMNccuDvYHck1PFSyG1aaateQPqgKZFBMgZkFZaXHIf9RV4MDW5xpTK2b/+qbwOg==", + "license": "Apache-2.0", + "dependencies": { + "@livekit/mutex": "1.1.1", + "@livekit/protocol": "1.50.4", + "events": "^3.3.0", + "jose": "^6.1.0", + "loglevel": "^1.9.2", + "machina": "^7.0.0", + "sdp-transform": "^2.15.0", + "tslib": "2.8.1", + "typed-emitter": "^2.1.0", + "webrtc-adapter": "9.0.6" + }, + "peerDependencies": { + "@types/dom-mediacapture-record": "^1" + } + }, + "node_modules/livekit-client/node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/livekit-server-sdk": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/livekit-server-sdk/-/livekit-server-sdk-2.18.0.tgz", @@ -4076,6 +4114,19 @@ "node": ">=8" } }, + "node_modules/loglevel": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", + "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" + } + }, "node_modules/lru-cache": { "version": "11.5.2", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", @@ -4085,6 +4136,15 @@ "node": "20 || >=22" } }, + "node_modules/machina": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/machina/-/machina-7.0.1.tgz", + "integrity": "sha512-0Xkgs8NeBvO2hW6bjJT8ewLWrMfh1haMPstaghpASWjXkLjRhtDiaJG1Kb7mslaOHxZYxpH7PIjpmirRVYbMMA==", + "license": "MIT", + "engines": { + "node": ">=22.22" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -4828,6 +4888,16 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "license": "ISC" }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -4866,6 +4936,21 @@ "node": ">=11.0.0" } }, + "node_modules/sdp": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.2.tgz", + "integrity": "sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==", + "license": "MIT" + }, + "node_modules/sdp-transform": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz", + "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==", + "license": "MIT", + "bin": { + "sdp-verify": "checker.js" + } + }, "node_modules/secure-json-parse": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", @@ -4945,6 +5030,15 @@ "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==", "license": "MIT" }, + "node_modules/simli-client": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/simli-client/-/simli-client-3.0.2.tgz", + "integrity": "sha512-Evfxj1hr7r1rtk/HOK7Ub7WZZQStkrRBFvCvWG/zukaZUw1QJcRO2bH/YKUpU2beaLfothTCb2XegTV+YlhRFw==", + "license": "MIT", + "dependencies": { + "livekit-client": "^2.15.15" + } + }, "node_modules/sonic-boom": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz", @@ -5092,6 +5186,15 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "license": "MIT", + "optionalDependencies": { + "rxjs": "*" + } + }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", @@ -5151,6 +5254,19 @@ "node": ">=12" } }, + "node_modules/webrtc-adapter": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.6.tgz", + "integrity": "sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==", + "license": "BSD-3-Clause", + "dependencies": { + "sdp": "^3.2.0" + }, + "engines": { + "node": ">=6.0.0", + "npm": ">=3.10.0" + } + }, "node_modules/whatwg-fetch": { "version": "3.6.20", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", diff --git a/package.json b/package.json index 423e18f..3f19f38 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "eventbuzz-concierge", "version": "1.0.0", - "description": "EventBuzz Virtual Concierge — independent Bedrock two-call chatbot service", + "description": "EventBuzz Virtual Concierge — Bedrock tool-calling chatbot service", "main": "server.js", "engines": { "node": ">=20" @@ -10,7 +10,8 @@ "start": "node server.js", "free-port": "node scripts/free-port.js", "dev": "node scripts/free-port.js && node --watch server.js", - "livekit-voice-worker": "node src/workers/livekitVoiceAgent.js start" + "livekit-voice-worker": "node src/workers/livekitVoiceAgent.js start", + "bundle:simli": "npx esbuild node_modules/simli-client/lib/client.ts --bundle --format=esm --platform=browser --target=es2020 --minify --outfile=public/embed/simli-client.js" }, "dependencies": { "@aws-sdk/client-bedrock-runtime": "^3.1030.0", @@ -26,6 +27,7 @@ "livekit-server-sdk": "^2.18.0", "mongoose": "^8.0.0", "nodemailer": "^6.9.7", - "qrcode": "^1.5.4" + "qrcode": "^1.5.4", + "simli-client": "^3.0.2" } } diff --git a/public/embed-avatar.html b/public/embed-avatar.html new file mode 100644 index 0000000..fe024ae --- /dev/null +++ b/public/embed-avatar.html @@ -0,0 +1,47 @@ + + +
+ + +
+