249 lines
9.2 KiB
TypeScript
249 lines
9.2 KiB
TypeScript
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
// The evaluation of the documents module's extraction: a small golden set of synthetic documents, a
|
|
// scorer, and thresholds kept in the repository. It is written against the `LineExtractor` port and
|
|
// imports nothing from an app alias, so the vitest suite (offline, against the fake gateway) and the
|
|
// promptfoo provider (live, against the real one) run the same cases through the same scorer.
|
|
//
|
|
// Three numbers, each over the whole set:
|
|
// recall of the lines a person would expect, the share the extractor proposed (same page, and
|
|
// the same label and value once whitespace, case and diacritics are folded)
|
|
// precision of the lines it proposed, the share that were expected
|
|
// verification of the lines it proposed, the share whose quote is on the page it cites. This is
|
|
// checked here again, from the case's own text, and is not read from the extractor's
|
|
// claim: a verification that trusted the verifier would measure nothing.
|
|
//
|
|
// A case also says which of its pages the extraction must leave out (a drawing, an empty page) and which
|
|
// rows the reader flags as repeating (a running header): a drawing read anyway adds lines nobody
|
|
// expected and costs precision, and a repeated row read anyway does the same.
|
|
//
|
|
// Every document is invented for this purpose: no real, customer or personal data.
|
|
|
|
import { readdirSync, readFileSync } from "node:fs"
|
|
import type { PageClass } from "../../modules/documents/contract/constants.js"
|
|
import type {
|
|
Extraction,
|
|
ExtractionInput,
|
|
LineExtractor,
|
|
ReadDocument,
|
|
SkippedPage,
|
|
} from "../../modules/documents/core/ports.js"
|
|
import type { StageCache } from "../../modules/documents/core/stage-cache.js"
|
|
import { foldForComparison, quoteIsOnPage } from "../../modules/documents/core/verify.js"
|
|
|
|
export interface GoldenPage {
|
|
readonly rows: readonly string[]
|
|
readonly source?: "text_layer" | "ocr"
|
|
readonly confidence?: number
|
|
/** What the reader made of the page; by default `scan` for a recognised page and `text` for a text layer. */
|
|
readonly class?: PageClass
|
|
/** The rows the reader flags as repeating on most pages of the document (headers, footers). */
|
|
readonly repeated?: readonly string[]
|
|
}
|
|
|
|
export interface ExpectedLine {
|
|
readonly page: number
|
|
readonly label: string
|
|
readonly value: string
|
|
}
|
|
|
|
export interface GoldenCase {
|
|
readonly id: string
|
|
readonly language: "en" | "lt"
|
|
readonly description: string
|
|
readonly pages: readonly GoldenPage[]
|
|
readonly expected: readonly ExpectedLine[]
|
|
/** The pages the extraction must leave out, 1-based, with the class that leaves each out. */
|
|
readonly skipped?: readonly SkippedPage[]
|
|
}
|
|
|
|
export interface Thresholds {
|
|
readonly recall: number
|
|
readonly precision: number
|
|
readonly verification: number
|
|
}
|
|
|
|
export interface EvalThresholds {
|
|
readonly offline: Thresholds
|
|
readonly live: Thresholds
|
|
}
|
|
|
|
const here = (path: string): URL => new URL(path, import.meta.url)
|
|
|
|
export const loadThresholds = (): EvalThresholds =>
|
|
JSON.parse(readFileSync(here("./thresholds.json"), "utf8")) as EvalThresholds
|
|
|
|
export function loadCases(): GoldenCase[] {
|
|
return readdirSync(here("./golden/"))
|
|
.filter((name) => name.endsWith(".json"))
|
|
.sort()
|
|
.map((name) => JSON.parse(readFileSync(here(`./golden/${name}`), "utf8")) as GoldenCase)
|
|
}
|
|
|
|
/**
|
|
* The OCR document a case stands for: one line per row, forty pixels apart, on an A4 page at 150 dpi,
|
|
* one block per line with the reader's id `p{n}-b{m}`, and the rows the case lists as repeated flagged.
|
|
*/
|
|
export function readDocumentOf(golden: GoldenCase): ReadDocument {
|
|
return {
|
|
engine: { name: "golden-set", version: "1" },
|
|
page_count: golden.pages.length,
|
|
pages: golden.pages.map((page, index) => ({
|
|
index,
|
|
width: 1240,
|
|
height: 1754,
|
|
class: page.class ?? (page.source === "ocr" ? "scan" : "text"),
|
|
source: page.source ?? "text_layer",
|
|
text: page.rows.join("\n"),
|
|
lines: page.rows.map((text, order) => ({
|
|
text,
|
|
bbox: [100, 100 + order * 40, 1100, 130 + order * 40] as const,
|
|
confidence: page.confidence ?? 0.97,
|
|
order,
|
|
repeated: page.repeated?.includes(text) ?? false,
|
|
})),
|
|
blocks: page.rows.map((_, order) => ({
|
|
id: `p${index + 1}-b${order + 1}`,
|
|
bbox: [90, 90 + order * 40, 1110, 140 + order * 40] as const,
|
|
order,
|
|
lines: [order],
|
|
})),
|
|
})),
|
|
}
|
|
}
|
|
|
|
/** No cache: an evaluation asks the model every time, which is the point of running one. */
|
|
export const noCache: StageCache = {
|
|
async get() {
|
|
return undefined
|
|
},
|
|
async put() {},
|
|
}
|
|
|
|
/** `pageImage` is how the case's pages are drawn: through the platform's page route offline, none live. */
|
|
export async function extractCase(
|
|
extractor: LineExtractor,
|
|
golden: GoldenCase,
|
|
pageImage: ExtractionInput["pageImage"] = async () => undefined,
|
|
): Promise<Extraction> {
|
|
return extractor.extract({ document: readDocumentOf(golden), pageImage, cache: noCache })
|
|
}
|
|
|
|
export interface CaseResult {
|
|
readonly id: string
|
|
readonly expected: number
|
|
readonly found: number
|
|
readonly matched: number
|
|
readonly onPage: number
|
|
/** The expected lines nobody proposed, and the proposed lines nobody expected, for the report. */
|
|
readonly missed: readonly string[]
|
|
readonly extra: readonly string[]
|
|
/** The pages the case says must be left out that were not, and the pages left out that the case did not say. */
|
|
readonly skippedWrongly: readonly string[]
|
|
}
|
|
|
|
const same = (a: string, b: string): boolean => foldForComparison(a) === foldForComparison(b)
|
|
const describeLine = (line: { page: number; label: string; value: string }): string =>
|
|
`p${line.page} ${line.label} = ${line.value}`
|
|
|
|
const describeSkipped = (skipped: SkippedPage): string => `p${skipped.page} ${skipped.pageClass}`
|
|
|
|
export function scoreCase(golden: GoldenCase, extraction: Extraction): CaseResult {
|
|
const found = extraction.lines
|
|
const pending = [...found]
|
|
const missed: string[] = []
|
|
let matched = 0
|
|
for (const wanted of golden.expected) {
|
|
const at = pending.findIndex(
|
|
(line) => line.page === wanted.page && same(line.label, wanted.label) && same(line.value, wanted.value),
|
|
)
|
|
if (at < 0) {
|
|
missed.push(describeLine(wanted))
|
|
continue
|
|
}
|
|
pending.splice(at, 1)
|
|
matched += 1
|
|
}
|
|
const onPage = found.filter((line) => {
|
|
const page = golden.pages[line.page - 1]
|
|
return page !== undefined && quoteIsOnPage(page.rows.join("\n"), line.quote)
|
|
}).length
|
|
return {
|
|
id: golden.id,
|
|
expected: golden.expected.length,
|
|
found: found.length,
|
|
matched,
|
|
onPage,
|
|
missed,
|
|
extra: pending.map(describeLine),
|
|
skippedWrongly: [
|
|
...(golden.skipped ?? [])
|
|
.filter(
|
|
(wanted) => !extraction.skipped.some((one) => one.page === wanted.page && one.pageClass === wanted.pageClass),
|
|
)
|
|
.map((one) => `not skipped: ${describeSkipped(one)}`),
|
|
...extraction.skipped
|
|
.filter(
|
|
(one) =>
|
|
!(golden.skipped ?? []).some((wanted) => wanted.page === one.page && wanted.pageClass === one.pageClass),
|
|
)
|
|
.map((one) => `skipped: ${describeSkipped(one)}`),
|
|
],
|
|
}
|
|
}
|
|
|
|
export interface Metrics extends Thresholds {
|
|
readonly expected: number
|
|
readonly found: number
|
|
}
|
|
|
|
const ratio = (part: number, whole: number): number => (whole === 0 ? 1 : part / whole)
|
|
|
|
export function aggregate(results: readonly CaseResult[]): Metrics {
|
|
const sum = (pick: (result: CaseResult) => number): number =>
|
|
results.reduce((total, result) => total + pick(result), 0)
|
|
return {
|
|
expected: sum((result) => result.expected),
|
|
found: sum((result) => result.found),
|
|
recall: ratio(
|
|
sum((result) => result.matched),
|
|
sum((result) => result.expected),
|
|
),
|
|
precision: ratio(
|
|
sum((result) => result.matched),
|
|
sum((result) => result.found),
|
|
),
|
|
verification: ratio(
|
|
sum((result) => result.onPage),
|
|
sum((result) => result.found),
|
|
),
|
|
}
|
|
}
|
|
|
|
/** What is below its threshold, as sentences; empty when the run passes. */
|
|
export function failuresOf(metrics: Thresholds, thresholds: Thresholds): string[] {
|
|
return (["recall", "precision", "verification"] as const)
|
|
.filter((name) => metrics[name] < thresholds[name])
|
|
.map(
|
|
(name) =>
|
|
`${name} ${(metrics[name] * 100).toFixed(1)}% is below the threshold ${(thresholds[name] * 100).toFixed(1)}%`,
|
|
)
|
|
}
|
|
|
|
export function report(results: readonly CaseResult[], metrics: Metrics): string {
|
|
const pct = (value: number): string => `${(value * 100).toFixed(1)}%`
|
|
const rows = results.map(
|
|
(result) =>
|
|
` ${result.id.padEnd(24)} expected ${String(result.expected).padStart(2)} found ${String(result.found).padStart(2)} matched ${String(result.matched).padStart(2)} quote on page ${result.onPage}/${result.found}` +
|
|
[
|
|
...result.missed.map((line) => `\n missed: ${line}`),
|
|
...result.extra.map((line) => `\n extra: ${line}`),
|
|
...result.skippedWrongly.map((line) => `\n pages: ${line}`),
|
|
].join(""),
|
|
)
|
|
return [
|
|
"documents extraction eval",
|
|
...rows,
|
|
` recall ${pct(metrics.recall)} precision ${pct(metrics.precision)} verification ${pct(metrics.verification)} (${metrics.found} proposed, ${metrics.expected} expected)`,
|
|
].join("\n")
|
|
}
|