227 lines
10 KiB
TypeScript
227 lines
10 KiB
TypeScript
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
// The documents extraction eval (plan item AT2.3), as a test.
|
|
//
|
|
// Offline, which is what `pnpm test` and CI run: the golden set goes through the real extractor and the
|
|
// real gateway client to the fake platform, whose model is the deterministic reader of
|
|
// test/support/fake-model.ts. A page the extractor wants to show the model is drawn through the fake
|
|
// platform's page route (`GET /v1/ocr/jobs/{id}/pages/{n}.png`) by the same OCR adapter the worker uses.
|
|
// That measures this module's code (request, schema, page plan, verification, scoring), against
|
|
// thresholds of 100%, and it must be able to fail, which the negative controls below prove.
|
|
//
|
|
// Live, `EVAL_LIVE=1 pnpm eval`: the same set and the same scorer against the platform's AI gateway
|
|
// (CONUSAI_API_URL, CONUSAI_API_TOKEN, and AI_MODEL for the model), against the live thresholds of
|
|
// thresholds.json. It costs credits and is not run by the gate.
|
|
|
|
import { readFileSync } from "node:fs"
|
|
import { aiModelOf } from "@app/config"
|
|
import type { ExtractionInput } from "@app/modules/documents"
|
|
import { LOW_TRUST_NOTE, modelLineExtractor, sdkDocumentOcr } from "@app/modules/documents"
|
|
import { createDocumentReader, createModelGateway, type ModelGateway } from "@conusai-kit/platform"
|
|
import { type FakeChatResponder, startFakePlatform } from "@conusai-kit/testing/fake-platform"
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest"
|
|
import { parse } from "yaml"
|
|
import { deterministicModel } from "../../test/support/fake-model.js"
|
|
import {
|
|
aggregate,
|
|
type CaseResult,
|
|
extractCase,
|
|
failuresOf,
|
|
type GoldenCase,
|
|
loadCases,
|
|
loadThresholds,
|
|
readDocumentOf,
|
|
report,
|
|
scoreCase,
|
|
} from "./harness.js"
|
|
|
|
const live = process.env.EVAL_LIVE === "1"
|
|
const cases = loadCases()
|
|
const thresholds = loadThresholds()
|
|
|
|
async function evaluate(
|
|
gateway: ModelGateway,
|
|
model: string,
|
|
pageImageFor?: (golden: GoldenCase) => Promise<ExtractionInput["pageImage"]>,
|
|
) {
|
|
const extractor = modelLineExtractor({ gateway, model })
|
|
const results: CaseResult[] = []
|
|
for (const golden of cases) {
|
|
const extraction = await extractCase(extractor, golden, pageImageFor ? await pageImageFor(golden) : undefined)
|
|
results.push(scoreCase(golden, extraction))
|
|
}
|
|
return { results, metrics: aggregate(results) }
|
|
}
|
|
|
|
describe("the golden set", () => {
|
|
it("is small, generic and in both languages", () => {
|
|
expect(cases.length).toBeGreaterThanOrEqual(5)
|
|
expect(new Set(cases.map((golden) => golden.id)).size).toBe(cases.length)
|
|
expect(new Set(cases.map((golden) => golden.language))).toEqual(new Set(["en", "lt"]))
|
|
})
|
|
|
|
it("is the set the promptfoo config runs", () => {
|
|
const config = parse(readFileSync(new URL("./promptfooconfig.yaml", import.meta.url), "utf8")) as {
|
|
tests: Array<{ vars: { case: string } }>
|
|
}
|
|
expect(config.tests.map((test) => test.vars.case).sort()).toEqual(cases.map((golden) => golden.id).sort())
|
|
})
|
|
|
|
it("has a scanned PDF with a drawing page to leave out and a scan page to read, in both languages", () => {
|
|
const withDrawing = cases.find((golden) => golden.skipped?.some((page) => page.pageClass === "drawing"))
|
|
expect(withDrawing?.pages.some((page) => page.class === "scan")).toBe(true)
|
|
expect(withDrawing?.expected.length).toBeGreaterThan(0)
|
|
expect(withDrawing?.expected.every((line) => line.page !== withDrawing.skipped?.[0]?.page)).toBe(true)
|
|
const scans = cases.filter((golden) => golden.pages.some((page) => page.class === "scan"))
|
|
expect(new Set(scans.map((golden) => golden.language))).toEqual(new Set(["en", "lt"]))
|
|
})
|
|
|
|
it("expects only lines that are on its own pages, so a perfect extractor scores 100%", () => {
|
|
for (const golden of cases) {
|
|
for (const wanted of golden.expected) {
|
|
const text = readDocumentOf(golden).pages[wanted.page - 1]?.text ?? ""
|
|
expect(text, `${golden.id}: ${wanted.label}`).toContain(wanted.label)
|
|
expect(text, `${golden.id}: ${wanted.label}`).toContain(wanted.value)
|
|
}
|
|
}
|
|
})
|
|
})
|
|
|
|
describe.skipIf(live)("offline, against the fake gateway", () => {
|
|
let platform: Awaited<ReturnType<typeof startFakePlatform>>
|
|
let gateway: ModelGateway
|
|
|
|
beforeAll(async () => {
|
|
platform = await startFakePlatform()
|
|
gateway = createModelGateway({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" })
|
|
})
|
|
afterAll(async () => {
|
|
await platform.close()
|
|
})
|
|
|
|
/** The requests the model saw, newest run only. */
|
|
let requests: Array<Record<string, unknown>> = []
|
|
|
|
/**
|
|
* Draws a case's pages the way the worker does: the case is a job of the fake platform (its document
|
|
* is the OCR document the case stands for), and a page is fetched from that job's page route.
|
|
*/
|
|
const pagesThroughPlatform = async (golden: GoldenCase): Promise<ExtractionInput["pageImage"]> => {
|
|
platform.ocr.setDocument({ schema_version: "conusai.ocr.document/v1", ...readDocumentOf(golden) })
|
|
const ocr = sdkDocumentOcr(
|
|
createDocumentReader({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" }),
|
|
{
|
|
intervalMs: 1,
|
|
},
|
|
)
|
|
const { jobId } = await ocr.submit(new Uint8Array([1]), { filename: `${golden.id}.pdf`, idempotencyKey: golden.id })
|
|
await ocr.wait(jobId)
|
|
return (index) => ocr.page(jobId, index + 1)
|
|
}
|
|
|
|
const run = async (responder: FakeChatResponder) => {
|
|
requests = []
|
|
platform.ai.respond((request) => {
|
|
requests.push(request)
|
|
return responder(request)
|
|
})
|
|
return evaluate(gateway, "eval-model", pagesThroughPlatform)
|
|
}
|
|
|
|
type Part = { type: string; text?: string }
|
|
const partsOf = (request: Record<string, unknown>): Part[] => {
|
|
const user = (request.messages as Array<{ role: string; content: string | Part[] }>).find(
|
|
(message) => message.role === "user",
|
|
)
|
|
return typeof user?.content === "string" ? [] : (user?.content ?? [])
|
|
}
|
|
|
|
it("scores line recall, precision and quote verification above the thresholds", async () => {
|
|
const { results, metrics } = await run(deterministicModel)
|
|
console.log(report(results, metrics))
|
|
expect(failuresOf(metrics, thresholds.offline)).toEqual([])
|
|
// A page skipped or read against the case's word is a failure even when the ratios pass.
|
|
expect(results.flatMap((result) => result.skippedWrongly)).toEqual([])
|
|
expect(metrics.expected).toBeGreaterThan(30)
|
|
})
|
|
|
|
it("draws the scanned pages through the platform's page route and sends them with their text, and nothing else", async () => {
|
|
await run(deterministicModel)
|
|
const scans = cases
|
|
.flatMap((golden) => golden.pages)
|
|
.filter((page) => (page.class ?? (page.source === "ocr" ? "scan" : "text")) === "scan")
|
|
expect(scans.length).toBeGreaterThanOrEqual(3)
|
|
const withImage = requests.filter((request) => partsOf(request).some((part) => part.type === "image_url"))
|
|
expect(withImage).toHaveLength(scans.length)
|
|
for (const request of withImage) expect(partsOf(request)[0]?.text).toContain(LOW_TRUST_NOTE)
|
|
|
|
const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as { ocrPages: unknown[] }
|
|
expect(state.ocrPages.length).toBeGreaterThanOrEqual(scans.length)
|
|
})
|
|
|
|
it("never asks about a drawing, an empty page or a repeated line", async () => {
|
|
await run(deterministicModel)
|
|
const texts = requests.map((request) => partsOf(request)[0]?.text ?? "")
|
|
expect(texts.some((text) => text.includes("A-100 2400"))).toBe(false)
|
|
expect(texts.some((text) => text.includes("Customer: Example Ltd"))).toBe(false)
|
|
expect(texts.some((text) => text.includes("Puslapio poraštė"))).toBe(false)
|
|
})
|
|
|
|
it("fails when a drawing is read: the lines nobody expected cost precision", async () => {
|
|
const reading = modelLineExtractor({ gateway, model: "eval-model", excludedPageClasses: [] })
|
|
platform.ai.respond(deterministicModel)
|
|
const results: CaseResult[] = []
|
|
for (const golden of cases) {
|
|
results.push(scoreCase(golden, await extractCase(reading, golden, await pagesThroughPlatform(golden))))
|
|
}
|
|
const metrics = aggregate(results)
|
|
expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
|
|
expect(results.flatMap((result) => result.skippedWrongly).join(" ")).toContain("not skipped")
|
|
})
|
|
|
|
it("fails when the model's quotes are not on the page: the verification is measured, not assumed", async () => {
|
|
// every digit of every quote shifted by one: plausible-looking lines whose words are not on the page
|
|
const { metrics } = await run((request) => {
|
|
const reply = deterministicModel(request)
|
|
const body = JSON.parse(reply.content ?? "{}") as { lines?: Array<{ quote: string }> }
|
|
for (const line of body.lines ?? [])
|
|
line.quote = line.quote.replace(/\d/g, (digit) => String((Number(digit) + 1) % 10))
|
|
return { content: JSON.stringify(body) }
|
|
})
|
|
expect(metrics.verification).toBeLessThan(thresholds.offline.verification)
|
|
expect(failuresOf(metrics, thresholds.offline).join(" ")).toContain("verification")
|
|
})
|
|
|
|
it("fails when the model finds nothing: the recall is measured", async () => {
|
|
const { metrics } = await run(() => ({ content: JSON.stringify({ lines: [] }) }))
|
|
expect(metrics.recall).toBe(0)
|
|
expect(failuresOf(metrics, thresholds.offline).join(" ")).toContain("recall")
|
|
})
|
|
|
|
it("fails when the model invents lines: the precision is measured", async () => {
|
|
const { metrics } = await run((request) => {
|
|
const reply = deterministicModel(request)
|
|
const body = JSON.parse(reply.content ?? "{}") as {
|
|
lines?: Array<{ label: string; value: string; quote: string }>
|
|
}
|
|
body.lines = [...(body.lines ?? []), { label: "Invented", value: "1", quote: "Invented: 1" }]
|
|
return { content: JSON.stringify(body) }
|
|
})
|
|
expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
|
|
})
|
|
})
|
|
|
|
describe.skipIf(!live)("live, against the platform's AI gateway", () => {
|
|
it("scores line recall, precision and quote verification above the live thresholds", async () => {
|
|
const url = process.env.CONUSAI_API_URL
|
|
const token = process.env.CONUSAI_API_TOKEN
|
|
if (!url || !token) throw new Error("EVAL_LIVE=1 needs CONUSAI_API_URL and CONUSAI_API_TOKEN")
|
|
const model = aiModelOf()
|
|
const { results, metrics } = await evaluate(
|
|
createModelGateway({ CONUSAI_API_URL: url, CONUSAI_API_TOKEN: token }, { model }),
|
|
model,
|
|
)
|
|
console.log(`model: ${model}\n${report(results, metrics)}`)
|
|
expect(failuresOf(metrics, thresholds.live)).toEqual([])
|
|
}, 600_000)
|
|
})
|