TalkingBot/ingestion/tests/stage1.test.js

const fs = require('fs/promises');
const os = require('os');
const path = require('path');
const { PageSchema, ChapterSummarySchema, BookCardSchema, createChapterSummarySchema } = require('../src/schemas');
const { loadConfig } = require('../src/config');
const { LocalFsStorage, shouldSkip } = require('../src/storage');
const { mapBookletStatus, ensureBookletColumns, LocalJsonBookStore, BOOKLET_COLUMNS } = require('../src/books');

describe('schemas', () => {
  it('rejects a page without text', () => {
    expect(() => PageSchema.parse({ page: 1, removedLines: [] })).toThrow();
  });

  it('accepts a page', () => {
    expect(PageSchema.parse({ page: 1, text: 'Hello', removedLines: [] }).page).toBe(1);
  });
});

describe('config', () => {
  it('allows a local run without AWS settings', () => {
    const config = loadConfig({ LOCAL_DATA_DIR: '/tmp/books' }, { local: true });
    expect(config.local).toBe(true);
    expect(config.chunkTargetTokens).toBe(400);
    expect(config.awsRegion).toBe('ap-south-1');
  });

  it('requires model and database settings outside local mode', () => {
    expect(() => loadConfig({})).toThrow(/Missing required environment/);
  });
});

describe('book status', () => {
  it('maps pipeline status onto the existing enum', () => {
    expect(mapBookletStatus('OCR_DONE')).toBe('processing');
    expect(mapBookletStatus('INDEXED')).toBe('processing');
    expect(mapBookletStatus('READY')).toBe('completed');
    expect(mapBookletStatus('FAILED')).toBe('failed');
  });

  it('ignores duplicate column errors', async () => {
    const queries = [];
    const pool = {
      async query(sql) {
        queries.push(sql);
        const error = new Error('duplicate');
        error.errno = 1060;
        throw error;
      },
    };
    await ensureBookletColumns(pool);
    expect(queries).toHaveLength(BOOKLET_COLUMNS.length);
  });
});

describe('local storage', () => {
  let dir;
  let storage;

  beforeEach(async () => {
    dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ingest-'));
    storage = new LocalFsStorage(dir);
  });

  afterEach(async () => {
    await fs.rm(dir, { recursive: true, force: true });
  });

  it('round-trips bytes, json, and text', async () => {
    await storage.putBytes('books/1/source.pdf', Buffer.from('pdf'));
    await storage.putJson('books/1/meta/stats.json', { pageCount: 2 });
    await storage.putText('books/1/clean/book_marked.txt', '[Page 1]\nHi\n');
    expect((await storage.getBytes('books/1/source.pdf')).toString()).toBe('pdf');
    expect(await storage.getJson('books/1/meta/stats.json')).toEqual({ pageCount: 2 });
    expect(await storage.getText('books/1/clean/book_marked.txt')).toContain('[Page 1]');
    expect(await storage.list('books/1/')).toEqual([
      'books/1/clean/book_marked.txt',
      'books/1/meta/stats.json',
      'books/1/source.pdf',
    ]);
  });

  it('skips an existing artifact unless force is set', async () => {
    await storage.putText('books/1/clean/pages.jsonl', '{}\n');
    expect(await shouldSkip(storage, 'books/1/clean/pages.jsonl', false)).toBe(true);
    expect(await shouldSkip(storage, 'books/1/clean/pages.jsonl', true)).toBe(false);
    expect(await shouldSkip(storage, 'books/1/missing', false)).toBe(false);
  });

  it('stores a local book record', async () => {
    const books = new LocalJsonBookStore(storage);
    await books.upsert(7, { ingestionStatus: 'CLEANED', pageCount: 4 });
    const row = await books.get(7);
    expect(row.status).toBe('processing');
    expect(row.ingestionStatus).toBe('CLEANED');
    expect(row.pageCount).toBe(4);
  });
});

describe('summary schemas', () => {
  it('rejects a short chapter summary', () => {
    expect(() =>
      ChapterSummarySchema.parse({
        chapterNo: 1,
        title: 'One',
        pageStart: 1,
        pageEnd: 2,
        oneLine: 'A line',
        summary: 'too short',
        keyPoints: ['a', 'b', 'c'],
        entities: { people: [], places: [], terms: [] },
        questionsAnswered: ['q1', 'q2', 'q3'],
      }),
    ).toThrow(/100-200/);
  });

  it('accepts a short summary when the chapter text is short', () => {
    const parsed = createChapterSummarySchema(1, 200).parse({
      chapterNo: 1,
      title: 'One',
      pageStart: 1,
      pageEnd: 2,
      oneLine: 'A line',
      summary: 'A short session note.',
      keyPoints: ['a', 'b', 'c'],
      entities: { people: [], places: [], terms: [] },
      questionsAnswered: ['q1', 'q2', 'q3'],
    });
    expect(parsed.summary).toBe('A short session note.');
  });

  it('rejects an invented-length book overview', () => {
    expect(() =>
      BookCardSchema.parse({
        title: null,
        author: null,
        genre: null,
        intendedAudience: null,
        language: null,
        themes: [],
        overview: 'short',
        chapters: [],
        glossary: [],
      }),
    ).toThrow(/150-250/);
  });
});