This commit is contained in:
ConusAI vendor v0.1.0-dev.20261027+9b1131fb
@@ -0,0 +1,248 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The evaluation of the documents module's extraction: a small golden set of synthetic documents, a
|
||||
// scorer, and thresholds kept in the repository. It is written against the `LineExtractor` port and
|
||||
// imports nothing from an app alias, so the vitest suite (offline, against the fake gateway) and the
|
||||
// promptfoo provider (live, against the real one) run the same cases through the same scorer.
|
||||
//
|
||||
// Three numbers, each over the whole set:
|
||||
// recall of the lines a person would expect, the share the extractor proposed (same page, and
|
||||
// the same label and value once whitespace, case and diacritics are folded)
|
||||
// precision of the lines it proposed, the share that were expected
|
||||
// verification of the lines it proposed, the share whose quote is on the page it cites. This is
|
||||
// checked here again, from the case's own text, and is not read from the extractor's
|
||||
// claim: a verification that trusted the verifier would measure nothing.
|
||||
//
|
||||
// A case also says which of its pages the extraction must leave out (a drawing, an empty page) and which
|
||||
// rows the reader flags as repeating (a running header): a drawing read anyway adds lines nobody
|
||||
// expected and costs precision, and a repeated row read anyway does the same.
|
||||
//
|
||||
// Every document is invented for this purpose: no real, customer or personal data.
|
||||
|
||||
import { readdirSync, readFileSync } from "node:fs"
|
||||
import type { PageClass } from "../../modules/documents/contract/constants.js"
|
||||
import type {
|
||||
Extraction,
|
||||
ExtractionInput,
|
||||
LineExtractor,
|
||||
ReadDocument,
|
||||
SkippedPage,
|
||||
} from "../../modules/documents/core/ports.js"
|
||||
import type { StageCache } from "../../modules/documents/core/stage-cache.js"
|
||||
import { foldForComparison, quoteIsOnPage } from "../../modules/documents/core/verify.js"
|
||||
|
||||
export interface GoldenPage {
|
||||
readonly rows: readonly string[]
|
||||
readonly source?: "text_layer" | "ocr"
|
||||
readonly confidence?: number
|
||||
/** What the reader made of the page; by default `scan` for a recognised page and `text` for a text layer. */
|
||||
readonly class?: PageClass
|
||||
/** The rows the reader flags as repeating on most pages of the document (headers, footers). */
|
||||
readonly repeated?: readonly string[]
|
||||
}
|
||||
|
||||
export interface ExpectedLine {
|
||||
readonly page: number
|
||||
readonly label: string
|
||||
readonly value: string
|
||||
}
|
||||
|
||||
export interface GoldenCase {
|
||||
readonly id: string
|
||||
readonly language: "en" | "lt"
|
||||
readonly description: string
|
||||
readonly pages: readonly GoldenPage[]
|
||||
readonly expected: readonly ExpectedLine[]
|
||||
/** The pages the extraction must leave out, 1-based, with the class that leaves each out. */
|
||||
readonly skipped?: readonly SkippedPage[]
|
||||
}
|
||||
|
||||
export interface Thresholds {
|
||||
readonly recall: number
|
||||
readonly precision: number
|
||||
readonly verification: number
|
||||
}
|
||||
|
||||
export interface EvalThresholds {
|
||||
readonly offline: Thresholds
|
||||
readonly live: Thresholds
|
||||
}
|
||||
|
||||
const here = (path: string): URL => new URL(path, import.meta.url)
|
||||
|
||||
export const loadThresholds = (): EvalThresholds =>
|
||||
JSON.parse(readFileSync(here("./thresholds.json"), "utf8")) as EvalThresholds
|
||||
|
||||
export function loadCases(): GoldenCase[] {
|
||||
return readdirSync(here("./golden/"))
|
||||
.filter((name) => name.endsWith(".json"))
|
||||
.sort()
|
||||
.map((name) => JSON.parse(readFileSync(here(`./golden/${name}`), "utf8")) as GoldenCase)
|
||||
}
|
||||
|
||||
/**
|
||||
* The OCR document a case stands for: one line per row, forty pixels apart, on an A4 page at 150 dpi,
|
||||
* one block per line with the reader's id `p{n}-b{m}`, and the rows the case lists as repeated flagged.
|
||||
*/
|
||||
export function readDocumentOf(golden: GoldenCase): ReadDocument {
|
||||
return {
|
||||
engine: { name: "golden-set", version: "1" },
|
||||
page_count: golden.pages.length,
|
||||
pages: golden.pages.map((page, index) => ({
|
||||
index,
|
||||
width: 1240,
|
||||
height: 1754,
|
||||
class: page.class ?? (page.source === "ocr" ? "scan" : "text"),
|
||||
source: page.source ?? "text_layer",
|
||||
text: page.rows.join("\n"),
|
||||
lines: page.rows.map((text, order) => ({
|
||||
text,
|
||||
bbox: [100, 100 + order * 40, 1100, 130 + order * 40] as const,
|
||||
confidence: page.confidence ?? 0.97,
|
||||
order,
|
||||
repeated: page.repeated?.includes(text) ?? false,
|
||||
})),
|
||||
blocks: page.rows.map((_, order) => ({
|
||||
id: `p${index + 1}-b${order + 1}`,
|
||||
bbox: [90, 90 + order * 40, 1110, 140 + order * 40] as const,
|
||||
order,
|
||||
lines: [order],
|
||||
})),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/** No cache: an evaluation asks the model every time, which is the point of running one. */
|
||||
export const noCache: StageCache = {
|
||||
async get() {
|
||||
return undefined
|
||||
},
|
||||
async put() {},
|
||||
}
|
||||
|
||||
/** `pageImage` is how the case's pages are drawn: through the platform's page route offline, none live. */
|
||||
export async function extractCase(
|
||||
extractor: LineExtractor,
|
||||
golden: GoldenCase,
|
||||
pageImage: ExtractionInput["pageImage"] = async () => undefined,
|
||||
): Promise<Extraction> {
|
||||
return extractor.extract({ document: readDocumentOf(golden), pageImage, cache: noCache })
|
||||
}
|
||||
|
||||
export interface CaseResult {
|
||||
readonly id: string
|
||||
readonly expected: number
|
||||
readonly found: number
|
||||
readonly matched: number
|
||||
readonly onPage: number
|
||||
/** The expected lines nobody proposed, and the proposed lines nobody expected, for the report. */
|
||||
readonly missed: readonly string[]
|
||||
readonly extra: readonly string[]
|
||||
/** The pages the case says must be left out that were not, and the pages left out that the case did not say. */
|
||||
readonly skippedWrongly: readonly string[]
|
||||
}
|
||||
|
||||
const same = (a: string, b: string): boolean => foldForComparison(a) === foldForComparison(b)
|
||||
const describeLine = (line: { page: number; label: string; value: string }): string =>
|
||||
`p${line.page} ${line.label} = ${line.value}`
|
||||
|
||||
const describeSkipped = (skipped: SkippedPage): string => `p${skipped.page} ${skipped.pageClass}`
|
||||
|
||||
export function scoreCase(golden: GoldenCase, extraction: Extraction): CaseResult {
|
||||
const found = extraction.lines
|
||||
const pending = [...found]
|
||||
const missed: string[] = []
|
||||
let matched = 0
|
||||
for (const wanted of golden.expected) {
|
||||
const at = pending.findIndex(
|
||||
(line) => line.page === wanted.page && same(line.label, wanted.label) && same(line.value, wanted.value),
|
||||
)
|
||||
if (at < 0) {
|
||||
missed.push(describeLine(wanted))
|
||||
continue
|
||||
}
|
||||
pending.splice(at, 1)
|
||||
matched += 1
|
||||
}
|
||||
const onPage = found.filter((line) => {
|
||||
const page = golden.pages[line.page - 1]
|
||||
return page !== undefined && quoteIsOnPage(page.rows.join("\n"), line.quote)
|
||||
}).length
|
||||
return {
|
||||
id: golden.id,
|
||||
expected: golden.expected.length,
|
||||
found: found.length,
|
||||
matched,
|
||||
onPage,
|
||||
missed,
|
||||
extra: pending.map(describeLine),
|
||||
skippedWrongly: [
|
||||
...(golden.skipped ?? [])
|
||||
.filter(
|
||||
(wanted) => !extraction.skipped.some((one) => one.page === wanted.page && one.pageClass === wanted.pageClass),
|
||||
)
|
||||
.map((one) => `not skipped: ${describeSkipped(one)}`),
|
||||
...extraction.skipped
|
||||
.filter(
|
||||
(one) =>
|
||||
!(golden.skipped ?? []).some((wanted) => wanted.page === one.page && wanted.pageClass === one.pageClass),
|
||||
)
|
||||
.map((one) => `skipped: ${describeSkipped(one)}`),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
export interface Metrics extends Thresholds {
|
||||
readonly expected: number
|
||||
readonly found: number
|
||||
}
|
||||
|
||||
const ratio = (part: number, whole: number): number => (whole === 0 ? 1 : part / whole)
|
||||
|
||||
export function aggregate(results: readonly CaseResult[]): Metrics {
|
||||
const sum = (pick: (result: CaseResult) => number): number =>
|
||||
results.reduce((total, result) => total + pick(result), 0)
|
||||
return {
|
||||
expected: sum((result) => result.expected),
|
||||
found: sum((result) => result.found),
|
||||
recall: ratio(
|
||||
sum((result) => result.matched),
|
||||
sum((result) => result.expected),
|
||||
),
|
||||
precision: ratio(
|
||||
sum((result) => result.matched),
|
||||
sum((result) => result.found),
|
||||
),
|
||||
verification: ratio(
|
||||
sum((result) => result.onPage),
|
||||
sum((result) => result.found),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/** What is below its threshold, as sentences; empty when the run passes. */
|
||||
export function failuresOf(metrics: Thresholds, thresholds: Thresholds): string[] {
|
||||
return (["recall", "precision", "verification"] as const)
|
||||
.filter((name) => metrics[name] < thresholds[name])
|
||||
.map(
|
||||
(name) =>
|
||||
`${name} ${(metrics[name] * 100).toFixed(1)}% is below the threshold ${(thresholds[name] * 100).toFixed(1)}%`,
|
||||
)
|
||||
}
|
||||
|
||||
export function report(results: readonly CaseResult[], metrics: Metrics): string {
|
||||
const pct = (value: number): string => `${(value * 100).toFixed(1)}%`
|
||||
const rows = results.map(
|
||||
(result) =>
|
||||
` ${result.id.padEnd(24)} expected ${String(result.expected).padStart(2)} found ${String(result.found).padStart(2)} matched ${String(result.matched).padStart(2)} quote on page ${result.onPage}/${result.found}` +
|
||||
[
|
||||
...result.missed.map((line) => `\n missed: ${line}`),
|
||||
...result.extra.map((line) => `\n extra: ${line}`),
|
||||
...result.skippedWrongly.map((line) => `\n pages: ${line}`),
|
||||
].join(""),
|
||||
)
|
||||
return [
|
||||
"documents extraction eval",
|
||||
...rows,
|
||||
` recall ${pct(metrics.recall)} precision ${pct(metrics.precision)} verification ${pct(metrics.verification)} (${metrics.found} proposed, ${metrics.expected} expected)`,
|
||||
].join("\n")
|
||||
}
|
||||
Reference in New Issue
Block a user