80 lines
2.9 KiB
JavaScript
80 lines
2.9 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Text-path check against a running backend and chat engine.
|
|
* Usage: node scripts/e2e-chat.mjs --base http://127.0.0.1:3000 --link TOKEN --token SESSION
|
|
* The link must already exist for a completed book. This script does not upload a PDF.
|
|
*/
|
|
const questions = [
|
|
{ kind: 'structural', text: 'How is this book organized?' },
|
|
{ kind: 'summary', text: 'Summarize the book in a few sentences.' },
|
|
{ kind: 'chapter', text: 'What happens in the first chapter?' },
|
|
{ kind: 'factual', text: 'Name one specific fact from the book and cite the page.' },
|
|
{ kind: 'unanswerable', text: 'What is the narrator\'s phone number?' },
|
|
];
|
|
|
|
function arg(name) {
|
|
const index = process.argv.indexOf(name);
|
|
return index === -1 ? '' : process.argv[index + 1];
|
|
}
|
|
|
|
async function readSse(response) {
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
let buffer = '';
|
|
const events = [];
|
|
while (true) {
|
|
const next = await reader.read();
|
|
if (next.done) break;
|
|
buffer += decoder.decode(next.value, { stream: true });
|
|
const parts = buffer.split('\n\n');
|
|
buffer = parts.pop() || '';
|
|
for (const part of parts) {
|
|
const line = part.split('\n').find((item) => item.startsWith('data:'));
|
|
if (line) events.push(JSON.parse(line.slice(5).trim()));
|
|
}
|
|
}
|
|
return events;
|
|
}
|
|
|
|
const base = arg('--base') || 'http://127.0.0.1:3000';
|
|
const link = arg('--link');
|
|
if (!link) {
|
|
process.stderr.write('Pass --link for an existing public link token.\n');
|
|
process.exitCode = 1;
|
|
} else {
|
|
const created = await fetch(`${base}/api/public/${link}/session`, { method: 'POST' });
|
|
const session = await created.json();
|
|
if (!created.ok) {
|
|
process.stderr.write(`${session.error || created.status}\n`);
|
|
process.exitCode = 1;
|
|
} else {
|
|
for (const question of questions) {
|
|
const response = await fetch(`${base}/api/public/sessions/${session.sessionId}/messages`, {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${session.sessionToken}`,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify({ text: question.text }),
|
|
});
|
|
const events = await readSse(response);
|
|
const finalEvent = events.find((event) => event.type === 'final');
|
|
const text = finalEvent ? finalEvent.text : '';
|
|
const citations = (finalEvent && finalEvent.citations) || [];
|
|
process.stdout.write(`${question.kind}: ${text.slice(0, 180).replace(/\n/g, ' ')}\n`);
|
|
if (question.kind === 'factual' && citations.length === 0) {
|
|
process.stderr.write('factual answer had no citation\n');
|
|
process.exitCode = 1;
|
|
}
|
|
if (!text) {
|
|
process.stderr.write(`${question.kind} answer was empty\n`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
await fetch(`${base}/api/public/sessions/${session.sessionId}/end`, {
|
|
method: 'POST',
|
|
headers: { Authorization: `Bearer ${session.sessionToken}` },
|
|
});
|
|
}
|
|
}
|