30 lines
1.1 KiB
YAML
30 lines
1.1 KiB
YAML
# SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
# The documents extraction eval, for promptfoo (https://www.promptfoo.dev), against the live AI gateway.
|
|
#
|
|
# CONUSAI_API_URL=... CONUSAI_API_TOKEN=... pnpm eval:promptfoo
|
|
#
|
|
# The provider runs the module's own extractor, so the prompt, the schema and the verification under
|
|
# test are the ones the worker runs. The offline gate that CI runs is documents.eval.test.ts, over the
|
|
# same golden set and the same scorer. To compare models, list the provider again with another
|
|
# `config.model`.
|
|
|
|
description: documents extraction, golden set
|
|
prompts:
|
|
- "{{case}}"
|
|
providers:
|
|
- id: file://promptfoo-provider.ts
|
|
label: default model
|
|
defaultTest:
|
|
assert:
|
|
- type: javascript
|
|
value: file://promptfoo-assert.ts
|
|
tests:
|
|
- vars: { case: invoice-en }
|
|
- vars: { case: invoice-lt }
|
|
- vars: { case: specification-en }
|
|
- vars: { case: specification-lt }
|
|
- vars: { case: delivery-note-scan-en }
|
|
- vars: { case: blank-page-en }
|
|
- vars: { case: scanned-pdf-drawing-en }
|
|
- vars: { case: scanned-pdf-lt }
|