ConusAI vendor v0.1.0-dev.20261027+9b1131fb

evals/documents/promptfoo-assert.ts

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The promptfoo assertion of the documents extraction eval: the scorer of harness.ts, applied to one
+// case's output, against the live thresholds of thresholds.json.
+
+import type { Extraction } from "../../modules/documents/core/ports.js"
+import { aggregate, failuresOf, loadCases, loadThresholds, scoreCase } from "./harness.js"
+
+interface GradingResult {
+  pass: boolean
+  score: number
+  reason: string
+}
+
+export default function assertCase(output: string, context: { vars?: Record<string, unknown> }): GradingResult {
+  const golden = loadCases().find((candidate) => candidate.id === String(context.vars?.case ?? ""))
+  if (!golden) return { pass: false, score: 0, reason: "no such golden case" }
+  const result = scoreCase(golden, JSON.parse(output) as Extraction)
+  const metrics = aggregate([result])
+  // The ratios against the thresholds, and the pages skipped or read against the case's word: a
+  // wrong skip fails the case even when recall, precision and verification all pass.
+  const failures = [...failuresOf(metrics, loadThresholds().live), ...result.skippedWrongly]
+  return {
+    pass: failures.length === 0,
+    score: (metrics.recall + metrics.precision + metrics.verification) / 3,
+    reason:
+      failures.length === 0
+        ? `recall ${metrics.recall.toFixed(2)}, precision ${metrics.precision.toFixed(2)}, verification ${metrics.verification.toFixed(2)}`
+        : [
+            ...failures,
+            ...result.missed.map((line) => `missed ${line}`),
+            ...result.extra.map((line) => `extra ${line}`),
+          ].join("; "),
+  }
+}