ConusAI vendor v0.1.0-dev.20261027+9b1131fb

evals/documents/documents.eval.test.ts

@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The documents extraction eval (plan item AT2.3), as a test.
+//
+// Offline, which is what `pnpm test` and CI run: the golden set goes through the real extractor and the
+// real gateway client to the fake platform, whose model is the deterministic reader of
+// test/support/fake-model.ts. A page the extractor wants to show the model is drawn through the fake
+// platform's page route (`GET /v1/ocr/jobs/{id}/pages/{n}.png`) by the same OCR adapter the worker uses.
+// That measures this module's code (request, schema, page plan, verification, scoring), against
+// thresholds of 100%, and it must be able to fail, which the negative controls below prove.
+//
+// Live, `EVAL_LIVE=1 pnpm eval`: the same set and the same scorer against the platform's AI gateway
+// (CONUSAI_API_URL, CONUSAI_API_TOKEN, and AI_MODEL for the model), against the live thresholds of
+// thresholds.json. It costs credits and is not run by the gate.
+
+import { readFileSync } from "node:fs"
+import { aiModelOf } from "@app/config"
+import type { ExtractionInput } from "@app/modules/documents"
+import { LOW_TRUST_NOTE, modelLineExtractor, sdkDocumentOcr } from "@app/modules/documents"
+import { createDocumentReader, createModelGateway, type ModelGateway } from "@conusai-kit/platform"
+import { type FakeChatResponder, startFakePlatform } from "@conusai-kit/testing/fake-platform"
+import { afterAll, beforeAll, describe, expect, it } from "vitest"
+import { parse } from "yaml"
+import { deterministicModel } from "../../test/support/fake-model.js"
+import {
+  aggregate,
+  type CaseResult,
+  extractCase,
+  failuresOf,
+  type GoldenCase,
+  loadCases,
+  loadThresholds,
+  readDocumentOf,
+  report,
+  scoreCase,
+} from "./harness.js"
+
+const live = process.env.EVAL_LIVE === "1"
+const cases = loadCases()
+const thresholds = loadThresholds()
+
+async function evaluate(
+  gateway: ModelGateway,
+  model: string,
+  pageImageFor?: (golden: GoldenCase) => Promise<ExtractionInput["pageImage"]>,
+) {
+  const extractor = modelLineExtractor({ gateway, model })
+  const results: CaseResult[] = []
+  for (const golden of cases) {
+    const extraction = await extractCase(extractor, golden, pageImageFor ? await pageImageFor(golden) : undefined)
+    results.push(scoreCase(golden, extraction))
+  }
+  return { results, metrics: aggregate(results) }
+}
+
+describe("the golden set", () => {
+  it("is small, generic and in both languages", () => {
+    expect(cases.length).toBeGreaterThanOrEqual(5)
+    expect(new Set(cases.map((golden) => golden.id)).size).toBe(cases.length)
+    expect(new Set(cases.map((golden) => golden.language))).toEqual(new Set(["en", "lt"]))
+  })
+
+  it("is the set the promptfoo config runs", () => {
+    const config = parse(readFileSync(new URL("./promptfooconfig.yaml", import.meta.url), "utf8")) as {
+      tests: Array<{ vars: { case: string } }>
+    }
+    expect(config.tests.map((test) => test.vars.case).sort()).toEqual(cases.map((golden) => golden.id).sort())
+  })
+
+  it("has a scanned PDF with a drawing page to leave out and a scan page to read, in both languages", () => {
+    const withDrawing = cases.find((golden) => golden.skipped?.some((page) => page.pageClass === "drawing"))
+    expect(withDrawing?.pages.some((page) => page.class === "scan")).toBe(true)
+    expect(withDrawing?.expected.length).toBeGreaterThan(0)
+    expect(withDrawing?.expected.every((line) => line.page !== withDrawing.skipped?.[0]?.page)).toBe(true)
+    const scans = cases.filter((golden) => golden.pages.some((page) => page.class === "scan"))
+    expect(new Set(scans.map((golden) => golden.language))).toEqual(new Set(["en", "lt"]))
+  })
+
+  it("expects only lines that are on its own pages, so a perfect extractor scores 100%", () => {
+    for (const golden of cases) {
+      for (const wanted of golden.expected) {
+        const text = readDocumentOf(golden).pages[wanted.page - 1]?.text ?? ""
+        expect(text, `${golden.id}: ${wanted.label}`).toContain(wanted.label)
+        expect(text, `${golden.id}: ${wanted.label}`).toContain(wanted.value)
+      }
+    }
+  })
+})
+
+describe.skipIf(live)("offline, against the fake gateway", () => {
+  let platform: Awaited<ReturnType<typeof startFakePlatform>>
+  let gateway: ModelGateway
+
+  beforeAll(async () => {
+    platform = await startFakePlatform()
+    gateway = createModelGateway({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" })
+  })
+  afterAll(async () => {
+    await platform.close()
+  })
+
+  /** The requests the model saw, newest run only. */
+  let requests: Array<Record<string, unknown>> = []
+
+  /**
+   * Draws a case's pages the way the worker does: the case is a job of the fake platform (its document
+   * is the OCR document the case stands for), and a page is fetched from that job's page route.
+   */
+  const pagesThroughPlatform = async (golden: GoldenCase): Promise<ExtractionInput["pageImage"]> => {
+    platform.ocr.setDocument({ schema_version: "conusai.ocr.document/v1", ...readDocumentOf(golden) })
+    const ocr = sdkDocumentOcr(
+      createDocumentReader({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" }),
+      {
+        intervalMs: 1,
+      },
+    )
+    const { jobId } = await ocr.submit(new Uint8Array([1]), { filename: `${golden.id}.pdf`, idempotencyKey: golden.id })
+    await ocr.wait(jobId)
+    return (index) => ocr.page(jobId, index + 1)
+  }
+
+  const run = async (responder: FakeChatResponder) => {
+    requests = []
+    platform.ai.respond((request) => {
+      requests.push(request)
+      return responder(request)
+    })
+    return evaluate(gateway, "eval-model", pagesThroughPlatform)
+  }
+
+  type Part = { type: string; text?: string }
+  const partsOf = (request: Record<string, unknown>): Part[] => {
+    const user = (request.messages as Array<{ role: string; content: string | Part[] }>).find(
+      (message) => message.role === "user",
+    )
+    return typeof user?.content === "string" ? [] : (user?.content ?? [])
+  }
+
+  it("scores line recall, precision and quote verification above the thresholds", async () => {
+    const { results, metrics } = await run(deterministicModel)
+    console.log(report(results, metrics))
+    expect(failuresOf(metrics, thresholds.offline)).toEqual([])
+    // A page skipped or read against the case's word is a failure even when the ratios pass.
+    expect(results.flatMap((result) => result.skippedWrongly)).toEqual([])
+    expect(metrics.expected).toBeGreaterThan(30)
+  })
+
+  it("draws the scanned pages through the platform's page route and sends them with their text, and nothing else", async () => {
+    await run(deterministicModel)
+    const scans = cases
+      .flatMap((golden) => golden.pages)
+      .filter((page) => (page.class ?? (page.source === "ocr" ? "scan" : "text")) === "scan")
+    expect(scans.length).toBeGreaterThanOrEqual(3)
+    const withImage = requests.filter((request) => partsOf(request).some((part) => part.type === "image_url"))
+    expect(withImage).toHaveLength(scans.length)
+    for (const request of withImage) expect(partsOf(request)[0]?.text).toContain(LOW_TRUST_NOTE)
+
+    const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as { ocrPages: unknown[] }
+    expect(state.ocrPages.length).toBeGreaterThanOrEqual(scans.length)
+  })
+
+  it("never asks about a drawing, an empty page or a repeated line", async () => {
+    await run(deterministicModel)
+    const texts = requests.map((request) => partsOf(request)[0]?.text ?? "")
+    expect(texts.some((text) => text.includes("A-100 2400"))).toBe(false)
+    expect(texts.some((text) => text.includes("Customer: Example Ltd"))).toBe(false)
+    expect(texts.some((text) => text.includes("Puslapio poraštė"))).toBe(false)
+  })
+
+  it("fails when a drawing is read: the lines nobody expected cost precision", async () => {
+    const reading = modelLineExtractor({ gateway, model: "eval-model", excludedPageClasses: [] })
+    platform.ai.respond(deterministicModel)
+    const results: CaseResult[] = []
+    for (const golden of cases) {
+      results.push(scoreCase(golden, await extractCase(reading, golden, await pagesThroughPlatform(golden))))
+    }
+    const metrics = aggregate(results)
+    expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
+    expect(results.flatMap((result) => result.skippedWrongly).join(" ")).toContain("not skipped")
+  })
+
+  it("fails when the model's quotes are not on the page: the verification is measured, not assumed", async () => {
+    // every digit of every quote shifted by one: plausible-looking lines whose words are not on the page
+    const { metrics } = await run((request) => {
+      const reply = deterministicModel(request)
+      const body = JSON.parse(reply.content ?? "{}") as { lines?: Array<{ quote: string }> }
+      for (const line of body.lines ?? [])
+        line.quote = line.quote.replace(/\d/g, (digit) => String((Number(digit) + 1) % 10))
+      return { content: JSON.stringify(body) }
+    })
+    expect(metrics.verification).toBeLessThan(thresholds.offline.verification)
+    expect(failuresOf(metrics, thresholds.offline).join(" ")).toContain("verification")
+  })
+
+  it("fails when the model finds nothing: the recall is measured", async () => {
+    const { metrics } = await run(() => ({ content: JSON.stringify({ lines: [] }) }))
+    expect(metrics.recall).toBe(0)
+    expect(failuresOf(metrics, thresholds.offline).join(" ")).toContain("recall")
+  })
+
+  it("fails when the model invents lines: the precision is measured", async () => {
+    const { metrics } = await run((request) => {
+      const reply = deterministicModel(request)
+      const body = JSON.parse(reply.content ?? "{}") as {
+        lines?: Array<{ label: string; value: string; quote: string }>
+      }
+      body.lines = [...(body.lines ?? []), { label: "Invented", value: "1", quote: "Invented: 1" }]
+      return { content: JSON.stringify(body) }
+    })
+    expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
+  })
+})
+
+describe.skipIf(!live)("live, against the platform's AI gateway", () => {
+  it("scores line recall, precision and quote verification above the live thresholds", async () => {
+    const url = process.env.CONUSAI_API_URL
+    const token = process.env.CONUSAI_API_TOKEN
+    if (!url || !token) throw new Error("EVAL_LIVE=1 needs CONUSAI_API_URL and CONUSAI_API_TOKEN")
+    const model = aiModelOf()
+    const { results, metrics } = await evaluate(
+      createModelGateway({ CONUSAI_API_URL: url, CONUSAI_API_TOKEN: token }, { model }),
+      model,
+    )
+    console.log(`model: ${model}\n${report(results, metrics)}`)
+    expect(failuresOf(metrics, thresholds.live)).toEqual([])
+  }, 600_000)
+})