51 lines
2.3 KiB
TypeScript
51 lines
2.3 KiB
TypeScript
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
// The promptfoo provider of the documents extraction eval: it runs one golden case through the real
|
|
// extractor and the real AI gateway, and returns the proposed lines as JSON for the assertions in
|
|
// promptfoo-assert.ts. Live only: promptfoo is run on demand (see evals/README.md), and the offline
|
|
// gate is documents.eval.test.ts.
|
|
//
|
|
// Configuration, from the environment: CONUSAI_API_URL and CONUSAI_API_TOKEN (the deployment
|
|
// contract), and the model, from the provider's own `config.model` in promptfooconfig.yaml, then
|
|
// AI_MODEL, then the app's default. Listing the same provider twice with two models compares them.
|
|
|
|
import { createModelGateway } from "@conusai-kit/platform"
|
|
import { aiModelOf } from "../../config/src/ai.js"
|
|
import { modelLineExtractor } from "../../modules/documents/adapters/model-extractor.js"
|
|
import { extractCase, loadCases } from "./harness.js"
|
|
|
|
interface ProviderOptions {
|
|
config?: { model?: string }
|
|
}
|
|
|
|
export default class DocumentsExtractionProvider {
|
|
private readonly model: string
|
|
|
|
constructor(options: ProviderOptions = {}) {
|
|
this.model = options.config?.model ?? aiModelOf()
|
|
}
|
|
|
|
id(): string {
|
|
return `documents-extraction:${this.model}`
|
|
}
|
|
|
|
async callApi(
|
|
_prompt: string,
|
|
context?: { vars?: Record<string, unknown> },
|
|
): Promise<{ output?: string; error?: string }> {
|
|
const url = process.env.CONUSAI_API_URL
|
|
const token = process.env.CONUSAI_API_TOKEN
|
|
if (!url || !token) return { error: "CONUSAI_API_URL and CONUSAI_API_TOKEN are required" }
|
|
const id = String(context?.vars?.case ?? "")
|
|
const golden = loadCases().find((candidate) => candidate.id === id)
|
|
if (!golden) return { error: `no golden case named "${id}"` }
|
|
try {
|
|
const gateway = createModelGateway({ CONUSAI_API_URL: url, CONUSAI_API_TOKEN: token }, { model: this.model })
|
|
// No page images live: the cases are rows of text, not files, so there is no job to draw a page from.
|
|
const extraction = await extractCase(modelLineExtractor({ gateway, model: this.model }), golden)
|
|
return { output: JSON.stringify(extraction) }
|
|
} catch (error) {
|
|
return { error: error instanceof Error ? `${error.name}: ${error.message}` : "the extraction failed" }
|
|
}
|
|
}
|
|
}
|