34 lines
1.4 KiB
TypeScript
34 lines
1.4 KiB
TypeScript
// The chunker: paragraph packing, hard splits of over-long paragraphs, positions a citation can
|
|
// point at. Pure paths — no store, no model.
|
|
|
|
import { describe, expect, it } from "vitest"
|
|
import { CHUNK_SIZE, chunkText } from "../core/chunk.js"
|
|
|
|
describe("chunkText", () => {
|
|
it("keeps a short document as one chunk", () => {
|
|
const chunks = chunkText("One paragraph.\n\nTwo paragraphs.")
|
|
expect(chunks).toHaveLength(1)
|
|
expect(chunks[0]?.position).toBe(1)
|
|
expect(chunks[0]?.text).toContain("Two paragraphs.")
|
|
})
|
|
|
|
it("splits on paragraph boundaries when the pack overflows the size", () => {
|
|
const paragraph = "word ".repeat(500) // 2500 characters
|
|
const chunks = chunkText(`${paragraph.trim()}\n\n${paragraph.trim()}`)
|
|
expect(chunks.length).toBeGreaterThanOrEqual(2)
|
|
for (const chunk of chunks) expect(chunk.text.length).toBeLessThanOrEqual(CHUNK_SIZE)
|
|
expect(chunks.map((chunk) => chunk.position)).toEqual(chunks.map((_, at) => at + 1))
|
|
})
|
|
|
|
it("hard-splits a paragraph longer than the size", () => {
|
|
const long = "x".repeat(CHUNK_SIZE * 2 + 100)
|
|
const chunks = chunkText(long)
|
|
expect(chunks.length).toBeGreaterThan(2)
|
|
expect(chunks.every((chunk) => chunk.text.length <= CHUNK_SIZE)).toBe(true)
|
|
})
|
|
|
|
it("refuses an empty document with a named field", () => {
|
|
expect(() => chunkText("")).toThrow(/nothing to index/)
|
|
})
|
|
})
|