114 lines
5.5 KiB
TypeScript
114 lines
5.5 KiB
TypeScript
// What code, not a model, decides about a line a model proposed: whether its quote is on the page it
|
|
// cites, where on the page it sits, how far the reader that produced those words can be trusted, and
|
|
// whether a page is one a model should be shown as an image.
|
|
//
|
|
// A model is asked for a verbatim quote because a quote can be checked. This is the check. A line
|
|
// whose quote is not found is not dropped (it may be a correct line with a misread quote) and is not
|
|
// trusted (it may be invented): it is kept, marked `needs_review`, and a person decides.
|
|
|
|
import type { FractionBox, PixelBox, ReadLine, ReadPage } from "./ports.js"
|
|
|
|
/**
|
|
* The form two strings are compared in, and used for nothing else: a quote is stored and shown as the
|
|
* model gave it. Whitespace runs collapse, case is folded, and combining marks are dropped after a
|
|
* canonical decomposition, so `Šarūnas` and `sarunas` meet. Nothing else is folded: punctuation, digits
|
|
* and letters that are different letters stay different, because a number read as another number is
|
|
* exactly the error this check exists to find.
|
|
*/
|
|
export function foldForComparison(text: string): string {
|
|
return text
|
|
.normalize("NFD")
|
|
.replace(/\p{M}+/gu, "")
|
|
.toLowerCase()
|
|
.replace(/\s+/gu, " ")
|
|
.trim()
|
|
}
|
|
|
|
/** True when the quote, folded, is a non-empty part of the page's text, folded. */
|
|
export function quoteIsOnPage(pageText: string, quote: string): boolean {
|
|
const wanted = foldForComparison(quote)
|
|
return wanted !== "" && foldForComparison(pageText).includes(wanted)
|
|
}
|
|
|
|
const clamp = (value: number): number => Math.min(1, Math.max(0, value))
|
|
|
|
export interface Location {
|
|
/** The union of the boxes of the blocks the quote runs over, as fractions of the page; without blocks, of the lines it falls on. None when the page has no size. */
|
|
readonly box?: FractionBox
|
|
/** The reader's id of the block holding the first line the quote falls on, when the reader grouped lines into blocks. */
|
|
readonly blockId?: string
|
|
/** The mean confidence of those lines. */
|
|
readonly confidence: number
|
|
}
|
|
|
|
function boxOf(page: ReadPage, boxes: readonly PixelBox[]): FractionBox | undefined {
|
|
if (!(page.width > 0) || !(page.height > 0) || boxes.length === 0) return undefined
|
|
const x0 = Math.min(...boxes.map((box) => box[0]))
|
|
const y0 = Math.min(...boxes.map((box) => box[1]))
|
|
const x1 = Math.max(...boxes.map((box) => box[2]))
|
|
const y1 = Math.max(...boxes.map((box) => box[3]))
|
|
const x = clamp(x0 / page.width)
|
|
const y = clamp(y0 / page.height)
|
|
// The box stays inside the page: a reader's box may overhang its edge by a pixel.
|
|
const width = Math.min(clamp((x1 - x0) / page.width), 1 - x)
|
|
const height = Math.min(clamp((y1 - y0) / page.height), 1 - y)
|
|
// A zero-area box is no locator: the proposal's evidence schema requires width and height > 0.
|
|
return width > 0 && height > 0 ? { x, y, width, height } : undefined
|
|
}
|
|
|
|
const mean = (values: readonly number[]): number =>
|
|
values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length
|
|
|
|
/**
|
|
* Where a quote sits on a page, from the reader's own lines: the lines it overlaps, in reading order.
|
|
* Undefined when the quote does not fall on the lines (the page's text and its lines disagree); the
|
|
* line is then still verified by the text, and simply has no box.
|
|
*/
|
|
export function locateQuote(page: ReadPage, quote: string): Location | undefined {
|
|
const wanted = foldForComparison(quote)
|
|
if (wanted === "") return undefined
|
|
const ordered = [...page.lines].sort((a, b) => a.order - b.order)
|
|
const spans: Array<{ line: ReadLine; start: number; end: number }> = []
|
|
let joined = ""
|
|
for (const line of ordered) {
|
|
const folded = foldForComparison(line.text)
|
|
if (folded === "") continue
|
|
if (joined !== "") joined += " "
|
|
spans.push({ line, start: joined.length, end: joined.length + folded.length })
|
|
joined += folded
|
|
}
|
|
const at = joined.indexOf(wanted)
|
|
if (at < 0) return undefined
|
|
const end = at + wanted.length
|
|
const hit = spans.filter((span) => span.start < end && span.end > at).map((span) => span.line)
|
|
if (hit.length === 0) return undefined
|
|
// A quote is located by the reader's blocks when it gave them: the id of the block holding its first
|
|
// line, and the boxes of the blocks it runs over, not a box made up from the lines it happened to fall on.
|
|
const indexes = hit.map((line) => page.lines.indexOf(line))
|
|
const blocks = page.blocks.filter((candidate) => indexes.some((index) => candidate.lines.includes(index)))
|
|
const first = blocks.find((candidate) => candidate.lines.includes(indexes[0] as number))
|
|
const box = boxOf(page, blocks.length > 0 ? blocks.map((block) => block.bbox) : hit.map((line) => line.bbox))
|
|
return {
|
|
...(box ? { box } : {}),
|
|
...(first ? { blockId: first.id } : {}),
|
|
confidence: clamp(mean(hit.map((line) => line.confidence))),
|
|
}
|
|
}
|
|
|
|
/** The mean confidence of a page's lines; 0 for a page with none. */
|
|
export const pageConfidence = (page: ReadPage): number => clamp(mean(page.lines.map((line) => line.confidence)))
|
|
|
|
/** Below this a page's text is a reading, not a record: a model is shown the image too. */
|
|
export const WEAK_PAGE_CONFIDENCE = 0.85
|
|
/** A page with less text than this has no text layer worth the name. */
|
|
export const MIN_PAGE_TEXT_CHARS = 20
|
|
|
|
/** A scan, a page the reader was unsure of, or a page with next to no text: one a model should see. */
|
|
export function isWeakPage(page: ReadPage): boolean {
|
|
return (
|
|
page.source === "ocr" ||
|
|
page.text.trim().length < MIN_PAGE_TEXT_CHARS ||
|
|
pageConfidence(page) < WEAK_PAGE_CONFIDENCE
|
|
)
|
|
}
|