ConusAI vendor v0.1.0-dev.20261027+9b1131fb

evals/documents/promptfoo-provider.ts

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The promptfoo provider of the documents extraction eval: it runs one golden case through the real
+// extractor and the real AI gateway, and returns the proposed lines as JSON for the assertions in
+// promptfoo-assert.ts. Live only: promptfoo is run on demand (see evals/README.md), and the offline
+// gate is documents.eval.test.ts.
+//
+// Configuration, from the environment: CONUSAI_API_URL and CONUSAI_API_TOKEN (the deployment
+// contract), and the model, from the provider's own `config.model` in promptfooconfig.yaml, then
+// AI_MODEL, then the app's default. Listing the same provider twice with two models compares them.
+
+import { createModelGateway } from "@conusai-kit/platform"
+import { aiModelOf } from "../../config/src/ai.js"
+import { modelLineExtractor } from "../../modules/documents/adapters/model-extractor.js"
+import { extractCase, loadCases } from "./harness.js"
+
+interface ProviderOptions {
+  config?: { model?: string }
+}
+
+export default class DocumentsExtractionProvider {
+  private readonly model: string
+
+  constructor(options: ProviderOptions = {}) {
+    this.model = options.config?.model ?? aiModelOf()
+  }
+
+  id(): string {
+    return `documents-extraction:${this.model}`
+  }
+
+  async callApi(
+    _prompt: string,
+    context?: { vars?: Record<string, unknown> },
+  ): Promise<{ output?: string; error?: string }> {
+    const url = process.env.CONUSAI_API_URL
+    const token = process.env.CONUSAI_API_TOKEN
+    if (!url || !token) return { error: "CONUSAI_API_URL and CONUSAI_API_TOKEN are required" }
+    const id = String(context?.vars?.case ?? "")
+    const golden = loadCases().find((candidate) => candidate.id === id)
+    if (!golden) return { error: `no golden case named "${id}"` }
+    try {
+      const gateway = createModelGateway({ CONUSAI_API_URL: url, CONUSAI_API_TOKEN: token }, { model: this.model })
+      // No page images live: the cases are rows of text, not files, so there is no job to draw a page from.
+      const extraction = await extractCase(modelLineExtractor({ gateway, model: this.model }), golden)
+      return { output: JSON.stringify(extraction) }
+    } catch (error) {
+      return { error: error instanceof Error ? `${error.name}: ${error.message}` : "the extraction failed" }
+    }
+  }
+}