ConusAI vendor v0.1.0-dev.20261027+9b1131fb

modules/knowledge/__tests__/chunk.test.ts

@@ -0,0 +1,33 @@
+// The chunker: paragraph packing, hard splits of over-long paragraphs, positions a citation can
+// point at. Pure paths — no store, no model.
+
+import { describe, expect, it } from "vitest"
+import { CHUNK_SIZE, chunkText } from "../core/chunk.js"
+
+describe("chunkText", () => {
+  it("keeps a short document as one chunk", () => {
+    const chunks = chunkText("One paragraph.\n\nTwo paragraphs.")
+    expect(chunks).toHaveLength(1)
+    expect(chunks[0]?.position).toBe(1)
+    expect(chunks[0]?.text).toContain("Two paragraphs.")
+  })
+
+  it("splits on paragraph boundaries when the pack overflows the size", () => {
+    const paragraph = "word ".repeat(500) // 2500 characters
+    const chunks = chunkText(`${paragraph.trim()}\n\n${paragraph.trim()}`)
+    expect(chunks.length).toBeGreaterThanOrEqual(2)
+    for (const chunk of chunks) expect(chunk.text.length).toBeLessThanOrEqual(CHUNK_SIZE)
+    expect(chunks.map((chunk) => chunk.position)).toEqual(chunks.map((_, at) => at + 1))
+  })
+
+  it("hard-splits a paragraph longer than the size", () => {
+    const long = "x".repeat(CHUNK_SIZE * 2 + 100)
+    const chunks = chunkText(long)
+    expect(chunks.length).toBeGreaterThan(2)
+    expect(chunks.every((chunk) => chunk.text.length <= CHUNK_SIZE)).toBe(true)
+  })
+
+  it("refuses an empty document with a named field", () => {
+    expect(() => chunkText("")).toThrow(/nothing to index/)
+  })
+})