// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The evaluation of the documents module's extraction: a small golden set of synthetic documents, a
// scorer, and thresholds kept in the repository. It is written against the `LineExtractor` port and
// imports nothing from an app alias, so the vitest suite (offline, against the fake gateway) and the
// promptfoo provider (live, against the real one) run the same cases through the same scorer.
//
// Three numbers, each over the whole set:
//   recall        of the lines a person would expect, the share the extractor proposed (same page, and
//                 the same label and value once whitespace, case and diacritics are folded)
//   precision     of the lines it proposed, the share that were expected
//   verification  of the lines it proposed, the share whose quote is on the page it cites. This is
//                 checked here again, from the case's own text, and is not read from the extractor's
//                 claim: a verification that trusted the verifier would measure nothing.
//
// A case also says which of its pages the extraction must leave out (a drawing, an empty page) and which
// rows the reader flags as repeating (a running header): a drawing read anyway adds lines nobody
// expected and costs precision, and a repeated row read anyway does the same.
//
// Every document is invented for this purpose: no real, customer or personal data.

import { readdirSync, readFileSync } from "node:fs"
import type { PageClass } from "../../modules/documents/contract/constants.js"
import type {
  Extraction,
  ExtractionInput,
  LineExtractor,
  ReadDocument,
  SkippedPage,
} from "../../modules/documents/core/ports.js"
import type { StageCache } from "../../modules/documents/core/stage-cache.js"
import { foldForComparison, quoteIsOnPage } from "../../modules/documents/core/verify.js"

export interface GoldenPage {
  readonly rows: readonly string[]
  readonly source?: "text_layer" | "ocr"
  readonly confidence?: number
  /** What the reader made of the page; by default `scan` for a recognised page and `text` for a text layer. */
  readonly class?: PageClass
  /** The rows the reader flags as repeating on most pages of the document (headers, footers). */
  readonly repeated?: readonly string[]
}

export interface ExpectedLine {
  readonly page: number
  readonly label: string
  readonly value: string
}

export interface GoldenCase {
  readonly id: string
  readonly language: "en" | "lt"
  readonly description: string
  readonly pages: readonly GoldenPage[]
  readonly expected: readonly ExpectedLine[]
  /** The pages the extraction must leave out, 1-based, with the class that leaves each out. */
  readonly skipped?: readonly SkippedPage[]
}

export interface Thresholds {
  readonly recall: number
  readonly precision: number
  readonly verification: number
}

export interface EvalThresholds {
  readonly offline: Thresholds
  readonly live: Thresholds
}

const here = (path: string): URL => new URL(path, import.meta.url)

export const loadThresholds = (): EvalThresholds =>
  JSON.parse(readFileSync(here("./thresholds.json"), "utf8")) as EvalThresholds

export function loadCases(): GoldenCase[] {
  return readdirSync(here("./golden/"))
    .filter((name) => name.endsWith(".json"))
    .sort()
    .map((name) => JSON.parse(readFileSync(here(`./golden/${name}`), "utf8")) as GoldenCase)
}

/**
 * The OCR document a case stands for: one line per row, forty pixels apart, on an A4 page at 150 dpi,
 * one block per line with the reader's id `p{n}-b{m}`, and the rows the case lists as repeated flagged.
 */
export function readDocumentOf(golden: GoldenCase): ReadDocument {
  return {
    engine: { name: "golden-set", version: "1" },
    page_count: golden.pages.length,
    pages: golden.pages.map((page, index) => ({
      index,
      width: 1240,
      height: 1754,
      class: page.class ?? (page.source === "ocr" ? "scan" : "text"),
      source: page.source ?? "text_layer",
      text: page.rows.join("\n"),
      lines: page.rows.map((text, order) => ({
        text,
        bbox: [100, 100 + order * 40, 1100, 130 + order * 40] as const,
        confidence: page.confidence ?? 0.97,
        order,
        repeated: page.repeated?.includes(text) ?? false,
      })),
      blocks: page.rows.map((_, order) => ({
        id: `p${index + 1}-b${order + 1}`,
        bbox: [90, 90 + order * 40, 1110, 140 + order * 40] as const,
        order,
        lines: [order],
      })),
    })),
  }
}

/** No cache: an evaluation asks the model every time, which is the point of running one. */
export const noCache: StageCache = {
  async get() {
    return undefined
  },
  async put() {},
}

/** `pageImage` is how the case's pages are drawn: through the platform's page route offline, none live. */
export async function extractCase(
  extractor: LineExtractor,
  golden: GoldenCase,
  pageImage: ExtractionInput["pageImage"] = async () => undefined,
): Promise<Extraction> {
  return extractor.extract({ document: readDocumentOf(golden), pageImage, cache: noCache })
}

export interface CaseResult {
  readonly id: string
  readonly expected: number
  readonly found: number
  readonly matched: number
  readonly onPage: number
  /** The expected lines nobody proposed, and the proposed lines nobody expected, for the report. */
  readonly missed: readonly string[]
  readonly extra: readonly string[]
  /** The pages the case says must be left out that were not, and the pages left out that the case did not say. */
  readonly skippedWrongly: readonly string[]
}

const same = (a: string, b: string): boolean => foldForComparison(a) === foldForComparison(b)
const describeLine = (line: { page: number; label: string; value: string }): string =>
  `p${line.page} ${line.label} = ${line.value}`

const describeSkipped = (skipped: SkippedPage): string => `p${skipped.page} ${skipped.pageClass}`

export function scoreCase(golden: GoldenCase, extraction: Extraction): CaseResult {
  const found = extraction.lines
  const pending = [...found]
  const missed: string[] = []
  let matched = 0
  for (const wanted of golden.expected) {
    const at = pending.findIndex(
      (line) => line.page === wanted.page && same(line.label, wanted.label) && same(line.value, wanted.value),
    )
    if (at < 0) {
      missed.push(describeLine(wanted))
      continue
    }
    pending.splice(at, 1)
    matched += 1
  }
  const onPage = found.filter((line) => {
    const page = golden.pages[line.page - 1]
    return page !== undefined && quoteIsOnPage(page.rows.join("\n"), line.quote)
  }).length
  return {
    id: golden.id,
    expected: golden.expected.length,
    found: found.length,
    matched,
    onPage,
    missed,
    extra: pending.map(describeLine),
    skippedWrongly: [
      ...(golden.skipped ?? [])
        .filter(
          (wanted) => !extraction.skipped.some((one) => one.page === wanted.page && one.pageClass === wanted.pageClass),
        )
        .map((one) => `not skipped: ${describeSkipped(one)}`),
      ...extraction.skipped
        .filter(
          (one) =>
            !(golden.skipped ?? []).some((wanted) => wanted.page === one.page && wanted.pageClass === one.pageClass),
        )
        .map((one) => `skipped: ${describeSkipped(one)}`),
    ],
  }
}

export interface Metrics extends Thresholds {
  readonly expected: number
  readonly found: number
}

const ratio = (part: number, whole: number): number => (whole === 0 ? 1 : part / whole)

export function aggregate(results: readonly CaseResult[]): Metrics {
  const sum = (pick: (result: CaseResult) => number): number =>
    results.reduce((total, result) => total + pick(result), 0)
  return {
    expected: sum((result) => result.expected),
    found: sum((result) => result.found),
    recall: ratio(
      sum((result) => result.matched),
      sum((result) => result.expected),
    ),
    precision: ratio(
      sum((result) => result.matched),
      sum((result) => result.found),
    ),
    verification: ratio(
      sum((result) => result.onPage),
      sum((result) => result.found),
    ),
  }
}

/** What is below its threshold, as sentences; empty when the run passes. */
export function failuresOf(metrics: Thresholds, thresholds: Thresholds): string[] {
  return (["recall", "precision", "verification"] as const)
    .filter((name) => metrics[name] < thresholds[name])
    .map(
      (name) =>
        `${name} ${(metrics[name] * 100).toFixed(1)}% is below the threshold ${(thresholds[name] * 100).toFixed(1)}%`,
    )
}

export function report(results: readonly CaseResult[], metrics: Metrics): string {
  const pct = (value: number): string => `${(value * 100).toFixed(1)}%`
  const rows = results.map(
    (result) =>
      `  ${result.id.padEnd(24)} expected ${String(result.expected).padStart(2)}  found ${String(result.found).padStart(2)}  matched ${String(result.matched).padStart(2)}  quote on page ${result.onPage}/${result.found}` +
      [
        ...result.missed.map((line) => `\n      missed: ${line}`),
        ...result.extra.map((line) => `\n      extra:  ${line}`),
        ...result.skippedWrongly.map((line) => `\n      pages:  ${line}`),
      ].join(""),
  )
  return [
    "documents extraction eval",
    ...rows,
    `  recall ${pct(metrics.recall)}  precision ${pct(metrics.precision)}  verification ${pct(metrics.verification)}  (${metrics.found} proposed, ${metrics.expected} expected)`,
  ].join("\n")
}