// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0 // The promptfoo provider of the documents extraction eval: it runs one golden case through the real // extractor and the real AI gateway, and returns the proposed lines as JSON for the assertions in // promptfoo-assert.ts. Live only: promptfoo is run on demand (see evals/README.md), and the offline // gate is documents.eval.test.ts. // // Configuration, from the environment: CONUSAI_API_URL and CONUSAI_API_TOKEN (the deployment // contract), and the model, from the provider's own `config.model` in promptfooconfig.yaml, then // AI_MODEL, then the app's default. Listing the same provider twice with two models compares them. import { createModelGateway } from "@conusai-kit/platform" import { aiModelOf } from "../../config/src/ai.js" import { modelLineExtractor } from "../../modules/documents/adapters/model-extractor.js" import { extractCase, loadCases } from "./harness.js" interface ProviderOptions { config?: { model?: string } } export default class DocumentsExtractionProvider { private readonly model: string constructor(options: ProviderOptions = {}) { this.model = options.config?.model ?? aiModelOf() } id(): string { return `documents-extraction:${this.model}` } async callApi( _prompt: string, context?: { vars?: Record }, ): Promise<{ output?: string; error?: string }> { const url = process.env.CONUSAI_API_URL const token = process.env.CONUSAI_API_TOKEN if (!url || !token) return { error: "CONUSAI_API_URL and CONUSAI_API_TOKEN are required" } const id = String(context?.vars?.case ?? "") const golden = loadCases().find((candidate) => candidate.id === id) if (!golden) return { error: `no golden case named "${id}"` } try { const gateway = createModelGateway({ CONUSAI_API_URL: url, CONUSAI_API_TOKEN: token }, { model: this.model }) // No page images live: the cases are rows of text, not files, so there is no job to draw a page from. const extraction = await extractCase(modelLineExtractor({ gateway, model: this.model }), golden) return { output: JSON.stringify(extraction) } } catch (error) { return { error: error instanceof Error ? `${error.name}: ${error.message}` : "the extraction failed" } } } }