39 lines
1.6 KiB
TypeScript
39 lines
1.6 KiB
TypeScript
// The chunker: one text into citation-sized pieces. Pure TypeScript, unit-tested without a store
|
|
// or a model. Chunks split on blank lines where they can, so a snippet reads as a sentence and not
|
|
// as a byte range; a chunk that fits stays whole.
|
|
|
|
import { validationFailed } from "@conusai-kit/core"
|
|
import type { KnowledgeChunk } from "./ports.js"
|
|
|
|
export const CHUNK_SIZE = 2_000
|
|
const CHUNK_OVERLAP = 200
|
|
|
|
/**
|
|
* Splits one document's text into chunks of at most {@link CHUNK_SIZE} characters. Paragraphs
|
|
* longer than the size are hard-split; the overlap keeps a sentence split across two chunks
|
|
* retrievable from either side.
|
|
*/
|
|
export function chunkText(text: string): KnowledgeChunk[] {
|
|
if (text.length === 0)
|
|
throw validationFailed([{ path: "file", message: "the file has no text to index" }], "nothing to index")
|
|
const paragraphs = text.split(/\r?\n\r?\n/).flatMap((paragraph) => {
|
|
if (paragraph.length <= CHUNK_SIZE) return [paragraph]
|
|
const pieces: string[] = []
|
|
for (let at = 0; at < paragraph.length; at += CHUNK_SIZE - CHUNK_OVERLAP) {
|
|
pieces.push(paragraph.slice(at, at + CHUNK_SIZE))
|
|
}
|
|
return pieces
|
|
})
|
|
const chunks: KnowledgeChunk[] = []
|
|
let current = ""
|
|
for (const paragraph of paragraphs) {
|
|
if (current !== "" && current.length + paragraph.length + 2 > CHUNK_SIZE) {
|
|
chunks.push({ position: chunks.length + 1, text: current })
|
|
current = ""
|
|
}
|
|
current = current === "" ? paragraph : `${current}\n\n${paragraph}`
|
|
}
|
|
if (current !== "") chunks.push({ position: chunks.length + 1, text: current })
|
|
return chunks
|
|
}
|