ConusAI vendor v0.1.0-dev.20261027+9b1131fb

modules/documents/core/verify.ts

@@ -0,0 +1,113 @@
+// What code, not a model, decides about a line a model proposed: whether its quote is on the page it
+// cites, where on the page it sits, how far the reader that produced those words can be trusted, and
+// whether a page is one a model should be shown as an image.
+//
+// A model is asked for a verbatim quote because a quote can be checked. This is the check. A line
+// whose quote is not found is not dropped (it may be a correct line with a misread quote) and is not
+// trusted (it may be invented): it is kept, marked `needs_review`, and a person decides.
+
+import type { FractionBox, PixelBox, ReadLine, ReadPage } from "./ports.js"
+
+/**
+ * The form two strings are compared in, and used for nothing else: a quote is stored and shown as the
+ * model gave it. Whitespace runs collapse, case is folded, and combining marks are dropped after a
+ * canonical decomposition, so `Šarūnas` and `sarunas` meet. Nothing else is folded: punctuation, digits
+ * and letters that are different letters stay different, because a number read as another number is
+ * exactly the error this check exists to find.
+ */
+export function foldForComparison(text: string): string {
+  return text
+    .normalize("NFD")
+    .replace(/\p{M}+/gu, "")
+    .toLowerCase()
+    .replace(/\s+/gu, " ")
+    .trim()
+}
+
+/** True when the quote, folded, is a non-empty part of the page's text, folded. */
+export function quoteIsOnPage(pageText: string, quote: string): boolean {
+  const wanted = foldForComparison(quote)
+  return wanted !== "" && foldForComparison(pageText).includes(wanted)
+}
+
+const clamp = (value: number): number => Math.min(1, Math.max(0, value))
+
+export interface Location {
+  /** The union of the boxes of the blocks the quote runs over, as fractions of the page; without blocks, of the lines it falls on. None when the page has no size. */
+  readonly box?: FractionBox
+  /** The reader's id of the block holding the first line the quote falls on, when the reader grouped lines into blocks. */
+  readonly blockId?: string
+  /** The mean confidence of those lines. */
+  readonly confidence: number
+}
+
+function boxOf(page: ReadPage, boxes: readonly PixelBox[]): FractionBox | undefined {
+  if (!(page.width > 0) || !(page.height > 0) || boxes.length === 0) return undefined
+  const x0 = Math.min(...boxes.map((box) => box[0]))
+  const y0 = Math.min(...boxes.map((box) => box[1]))
+  const x1 = Math.max(...boxes.map((box) => box[2]))
+  const y1 = Math.max(...boxes.map((box) => box[3]))
+  const x = clamp(x0 / page.width)
+  const y = clamp(y0 / page.height)
+  // The box stays inside the page: a reader's box may overhang its edge by a pixel.
+  const width = Math.min(clamp((x1 - x0) / page.width), 1 - x)
+  const height = Math.min(clamp((y1 - y0) / page.height), 1 - y)
+  // A zero-area box is no locator: the proposal's evidence schema requires width and height > 0.
+  return width > 0 && height > 0 ? { x, y, width, height } : undefined
+}
+
+const mean = (values: readonly number[]): number =>
+  values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length
+
+/**
+ * Where a quote sits on a page, from the reader's own lines: the lines it overlaps, in reading order.
+ * Undefined when the quote does not fall on the lines (the page's text and its lines disagree); the
+ * line is then still verified by the text, and simply has no box.
+ */
+export function locateQuote(page: ReadPage, quote: string): Location | undefined {
+  const wanted = foldForComparison(quote)
+  if (wanted === "") return undefined
+  const ordered = [...page.lines].sort((a, b) => a.order - b.order)
+  const spans: Array<{ line: ReadLine; start: number; end: number }> = []
+  let joined = ""
+  for (const line of ordered) {
+    const folded = foldForComparison(line.text)
+    if (folded === "") continue
+    if (joined !== "") joined += " "
+    spans.push({ line, start: joined.length, end: joined.length + folded.length })
+    joined += folded
+  }
+  const at = joined.indexOf(wanted)
+  if (at < 0) return undefined
+  const end = at + wanted.length
+  const hit = spans.filter((span) => span.start < end && span.end > at).map((span) => span.line)
+  if (hit.length === 0) return undefined
+  // A quote is located by the reader's blocks when it gave them: the id of the block holding its first
+  // line, and the boxes of the blocks it runs over, not a box made up from the lines it happened to fall on.
+  const indexes = hit.map((line) => page.lines.indexOf(line))
+  const blocks = page.blocks.filter((candidate) => indexes.some((index) => candidate.lines.includes(index)))
+  const first = blocks.find((candidate) => candidate.lines.includes(indexes[0] as number))
+  const box = boxOf(page, blocks.length > 0 ? blocks.map((block) => block.bbox) : hit.map((line) => line.bbox))
+  return {
+    ...(box ? { box } : {}),
+    ...(first ? { blockId: first.id } : {}),
+    confidence: clamp(mean(hit.map((line) => line.confidence))),
+  }
+}
+
+/** The mean confidence of a page's lines; 0 for a page with none. */
+export const pageConfidence = (page: ReadPage): number => clamp(mean(page.lines.map((line) => line.confidence)))
+
+/** Below this a page's text is a reading, not a record: a model is shown the image too. */
+export const WEAK_PAGE_CONFIDENCE = 0.85
+/** A page with less text than this has no text layer worth the name. */
+export const MIN_PAGE_TEXT_CHARS = 20
+
+/** A scan, a page the reader was unsure of, or a page with next to no text: one a model should see. */
+export function isWeakPage(page: ReadPage): boolean {
+  return (
+    page.source === "ocr" ||
+    page.text.trim().length < MIN_PAGE_TEXT_CHARS ||
+    pageConfidence(page) < WEAK_PAGE_CONFIDENCE
+  )
+}