61 lines
3.0 KiB
JavaScript
61 lines
3.0 KiB
JavaScript
const { summarizeReport, formatReport, measureMarkers, citationPrecisionRecall, voiceStyleReport } = require('../src/eval/run');
|
|
const questions = require('../src/eval/questions.json');
|
|
|
|
describe('eval report', () => {
|
|
it('averages score, latency, cost, and cache hits without calling AWS', () => {
|
|
const rows = [
|
|
{ mode: 'FULL_CONTEXT', category: 'factual', totalMs: 100, estimatedCostUsd: 0.2, cacheReadTokens: 10, score: 4 },
|
|
{ mode: 'FULL_CONTEXT', category: 'factual', totalMs: 300, estimatedCostUsd: 0.4, cacheReadTokens: 0, score: 2 },
|
|
{ mode: 'HYBRID_AGENTIC', category: 'factual', totalMs: 50, estimatedCostUsd: 0.1, cacheReadTokens: 0, score: 5 },
|
|
];
|
|
const report = summarizeReport(rows);
|
|
const full = report.byCategory.find((row) => row.mode === 'FULL_CONTEXT');
|
|
expect(full.score).toBe(3);
|
|
expect(full.p50Ms).toBe(100);
|
|
expect(full.cacheHitRate).toBe(0.5);
|
|
expect(report.sessionProjection.find((row) => row.mode === 'FULL_CONTEXT').costPer10TurnSession).toBeCloseTo(3);
|
|
expect(formatReport(report)).toContain('FULL_CONTEXT');
|
|
});
|
|
|
|
it('counts markers on the raw text and keeps prose reference points out of precision', () => {
|
|
const measured = measureMarkers({
|
|
rawText: 'See [[p:2]] and [[nope]].',
|
|
text: 'See and.',
|
|
ttsInput: 'See and.',
|
|
citations: [{ page: 2, verified: true }, { page: 4, verified: false }],
|
|
});
|
|
expect(measured).toMatchObject({
|
|
markers_emitted: 2,
|
|
markers_valid: 1,
|
|
markers_verified: 1,
|
|
malformed_markers: 1,
|
|
leaked_marker: 0,
|
|
});
|
|
expect(measureMarkers({ rawText: '[[p:2]]', text: 'Still [[p:2]]', ttsInput: 'Still [[p:2]]' }).leaked_marker).toBe(1);
|
|
expect(citationPrecisionRecall([{ page: 2 }], ['names the chapters in order']).precision).toBeNull();
|
|
expect(citationPrecisionRecall([{ page: 2, chapterNo: 1 }, { page: 9 }], ['on page 2'])).toEqual({
|
|
precision: 0.5,
|
|
recall: 1,
|
|
});
|
|
expect(voiceStyleReport('In chapter three.').pass).toBe(true);
|
|
expect(voiceStyleReport('In chapter three.').sentences).toBe(1);
|
|
expect(voiceStyleReport('* item').pass).toBe(false);
|
|
const report = summarizeReport([
|
|
{ mode: 'FULL_CONTEXT', category: 'factual', totalMs: 1, estimatedCostUsd: 0, outputTokens: 20, score: 1 },
|
|
{ mode: 'FULL_CONTEXT', category: 'factual', totalMs: 1, estimatedCostUsd: 0, outputTokens: 40, score: 1 },
|
|
]);
|
|
expect(report.averageOutputTokens).toBe(30);
|
|
const compared = { ...report, outputTokenDelta: report.averageOutputTokens - 10 };
|
|
expect(formatReport(compared)).toContain('output token delta: 20.0');
|
|
});
|
|
|
|
it('seeds five questions in each category for book 3', () => {
|
|
const categories = ['structural', 'summary', 'chapter', 'factual', 'multihop', 'unanswerable', 'followup'];
|
|
for (const category of categories) {
|
|
const rows = questions.filter((question) => question.category === category && question.bookId === '3');
|
|
expect(rows).toHaveLength(5);
|
|
expect(rows.every((row) => row.reference_points.length > 0)).toBe(true);
|
|
}
|
|
});
|
|
});
|