// The chunker: paragraph packing, hard splits of over-long paragraphs, positions a citation can // point at. Pure paths — no store, no model. import { describe, expect, it } from "vitest" import { CHUNK_SIZE, chunkText } from "../core/chunk.js" describe("chunkText", () => { it("keeps a short document as one chunk", () => { const chunks = chunkText("One paragraph.\n\nTwo paragraphs.") expect(chunks).toHaveLength(1) expect(chunks[0]?.position).toBe(1) expect(chunks[0]?.text).toContain("Two paragraphs.") }) it("splits on paragraph boundaries when the pack overflows the size", () => { const paragraph = "word ".repeat(500) // 2500 characters const chunks = chunkText(`${paragraph.trim()}\n\n${paragraph.trim()}`) expect(chunks.length).toBeGreaterThanOrEqual(2) for (const chunk of chunks) expect(chunk.text.length).toBeLessThanOrEqual(CHUNK_SIZE) expect(chunks.map((chunk) => chunk.position)).toEqual(chunks.map((_, at) => at + 1)) }) it("hard-splits a paragraph longer than the size", () => { const long = "x".repeat(CHUNK_SIZE * 2 + 100) const chunks = chunkText(long) expect(chunks.length).toBeGreaterThan(2) expect(chunks.every((chunk) => chunk.text.length <= CHUNK_SIZE)).toBe(true) }) it("refuses an empty document with a named field", () => { expect(() => chunkText("")).toThrow(/nothing to index/) }) })