ConusAI vendor v0.1.0-dev.20261027+9b1131fb

modules/knowledge/core/chunk.ts

@@ -0,0 +1,38 @@
+// The chunker: one text into citation-sized pieces. Pure TypeScript, unit-tested without a store
+// or a model. Chunks split on blank lines where they can, so a snippet reads as a sentence and not
+// as a byte range; a chunk that fits stays whole.
+
+import { validationFailed } from "@conusai-kit/core"
+import type { KnowledgeChunk } from "./ports.js"
+
+export const CHUNK_SIZE = 2_000
+const CHUNK_OVERLAP = 200
+
+/**
+ * Splits one document's text into chunks of at most {@link CHUNK_SIZE} characters. Paragraphs
+ * longer than the size are hard-split; the overlap keeps a sentence split across two chunks
+ * retrievable from either side.
+ */
+export function chunkText(text: string): KnowledgeChunk[] {
+  if (text.length === 0)
+    throw validationFailed([{ path: "file", message: "the file has no text to index" }], "nothing to index")
+  const paragraphs = text.split(/\r?\n\r?\n/).flatMap((paragraph) => {
+    if (paragraph.length <= CHUNK_SIZE) return [paragraph]
+    const pieces: string[] = []
+    for (let at = 0; at < paragraph.length; at += CHUNK_SIZE - CHUNK_OVERLAP) {
+      pieces.push(paragraph.slice(at, at + CHUNK_SIZE))
+    }
+    return pieces
+  })
+  const chunks: KnowledgeChunk[] = []
+  let current = ""
+  for (const paragraph of paragraphs) {
+    if (current !== "" && current.length + paragraph.length + 2 > CHUNK_SIZE) {
+      chunks.push({ position: chunks.length + 1, text: current })
+      current = ""
+    }
+    current = current === "" ? paragraph : `${current}\n\n${paragraph}`
+  }
+  if (current !== "") chunks.push({ position: chunks.length + 1, text: current })
+  return chunks
+}