// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The documents extraction eval (plan item AT2.3), as a test.
//
// Offline, which is what `pnpm test` and CI run: the golden set goes through the real extractor and the
// real gateway client to the fake platform, whose model is the deterministic reader of
// test/support/fake-model.ts. A page the extractor wants to show the model is drawn through the fake
// platform's page route (`GET /v1/ocr/jobs/{id}/pages/{n}.png`) by the same OCR adapter the worker uses.
// That measures this module's code (request, schema, page plan, verification, scoring), against
// thresholds of 100%, and it must be able to fail, which the negative controls below prove.
//
// Live, `EVAL_LIVE=1 pnpm eval`: the same set and the same scorer against the platform's AI gateway
// (CONUSAI_API_URL, CONUSAI_API_TOKEN, and AI_MODEL for the model), against the live thresholds of
// thresholds.json. It costs credits and is not run by the gate.

import { readFileSync } from "node:fs"
import { aiModelOf } from "@app/config"
import type { ExtractionInput } from "@app/modules/documents"
import { LOW_TRUST_NOTE, modelLineExtractor, sdkDocumentOcr } from "@app/modules/documents"
import { createDocumentReader, createModelGateway, type ModelGateway } from "@conusai-kit/platform"
import { type FakeChatResponder, startFakePlatform } from "@conusai-kit/testing/fake-platform"
import { afterAll, beforeAll, describe, expect, it } from "vitest"
import { parse } from "yaml"
import { deterministicModel } from "../../test/support/fake-model.js"
import {
  aggregate,
  type CaseResult,
  extractCase,
  failuresOf,
  type GoldenCase,
  loadCases,
  loadThresholds,
  readDocumentOf,
  report,
  scoreCase,
} from "./harness.js"

const live = process.env.EVAL_LIVE === "1"
const cases = loadCases()
const thresholds = loadThresholds()

async function evaluate(
  gateway: ModelGateway,
  model: string,
  pageImageFor?: (golden: GoldenCase) => Promise<ExtractionInput["pageImage"]>,
) {
  const extractor = modelLineExtractor({ gateway, model })
  const results: CaseResult[] = []
  for (const golden of cases) {
    const extraction = await extractCase(extractor, golden, pageImageFor ? await pageImageFor(golden) : undefined)
    results.push(scoreCase(golden, extraction))
  }
  return { results, metrics: aggregate(results) }
}

describe("the golden set", () => {
  it("is small, generic and in both languages", () => {
    expect(cases.length).toBeGreaterThanOrEqual(5)
    expect(new Set(cases.map((golden) => golden.id)).size).toBe(cases.length)
    expect(new Set(cases.map((golden) => golden.language))).toEqual(new Set(["en", "lt"]))
  })

  it("is the set the promptfoo config runs", () => {
    const config = parse(readFileSync(new URL("./promptfooconfig.yaml", import.meta.url), "utf8")) as {
      tests: Array<{ vars: { case: string } }>
    }
    expect(config.tests.map((test) => test.vars.case).sort()).toEqual(cases.map((golden) => golden.id).sort())
  })

  it("has a scanned PDF with a drawing page to leave out and a scan page to read, in both languages", () => {
    const withDrawing = cases.find((golden) => golden.skipped?.some((page) => page.pageClass === "drawing"))
    expect(withDrawing?.pages.some((page) => page.class === "scan")).toBe(true)
    expect(withDrawing?.expected.length).toBeGreaterThan(0)
    expect(withDrawing?.expected.every((line) => line.page !== withDrawing.skipped?.[0]?.page)).toBe(true)
    const scans = cases.filter((golden) => golden.pages.some((page) => page.class === "scan"))
    expect(new Set(scans.map((golden) => golden.language))).toEqual(new Set(["en", "lt"]))
  })

  it("expects only lines that are on its own pages, so a perfect extractor scores 100%", () => {
    for (const golden of cases) {
      for (const wanted of golden.expected) {
        const text = readDocumentOf(golden).pages[wanted.page - 1]?.text ?? ""
        expect(text, `${golden.id}: ${wanted.label}`).toContain(wanted.label)
        expect(text, `${golden.id}: ${wanted.label}`).toContain(wanted.value)
      }
    }
  })
})

describe.skipIf(live)("offline, against the fake gateway", () => {
  let platform: Awaited<ReturnType<typeof startFakePlatform>>
  let gateway: ModelGateway

  beforeAll(async () => {
    platform = await startFakePlatform()
    gateway = createModelGateway({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" })
  })
  afterAll(async () => {
    await platform.close()
  })

  /** The requests the model saw, newest run only. */
  let requests: Array<Record<string, unknown>> = []

  /**
   * Draws a case's pages the way the worker does: the case is a job of the fake platform (its document
   * is the OCR document the case stands for), and a page is fetched from that job's page route.
   */
  const pagesThroughPlatform = async (golden: GoldenCase): Promise<ExtractionInput["pageImage"]> => {
    platform.ocr.setDocument({ schema_version: "conusai.ocr.document/v1", ...readDocumentOf(golden) })
    const ocr = sdkDocumentOcr(
      createDocumentReader({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" }),
      {
        intervalMs: 1,
      },
    )
    const { jobId } = await ocr.submit(new Uint8Array([1]), { filename: `${golden.id}.pdf`, idempotencyKey: golden.id })
    await ocr.wait(jobId)
    return (index) => ocr.page(jobId, index + 1)
  }

  const run = async (responder: FakeChatResponder) => {
    requests = []
    platform.ai.respond((request) => {
      requests.push(request)
      return responder(request)
    })
    return evaluate(gateway, "eval-model", pagesThroughPlatform)
  }

  type Part = { type: string; text?: string }
  const partsOf = (request: Record<string, unknown>): Part[] => {
    const user = (request.messages as Array<{ role: string; content: string | Part[] }>).find(
      (message) => message.role === "user",
    )
    return typeof user?.content === "string" ? [] : (user?.content ?? [])
  }

  it("scores line recall, precision and quote verification above the thresholds", async () => {
    const { results, metrics } = await run(deterministicModel)
    console.log(report(results, metrics))
    expect(failuresOf(metrics, thresholds.offline)).toEqual([])
    // A page skipped or read against the case's word is a failure even when the ratios pass.
    expect(results.flatMap((result) => result.skippedWrongly)).toEqual([])
    expect(metrics.expected).toBeGreaterThan(30)
  })

  it("draws the scanned pages through the platform's page route and sends them with their text, and nothing else", async () => {
    await run(deterministicModel)
    const scans = cases
      .flatMap((golden) => golden.pages)
      .filter((page) => (page.class ?? (page.source === "ocr" ? "scan" : "text")) === "scan")
    expect(scans.length).toBeGreaterThanOrEqual(3)
    const withImage = requests.filter((request) => partsOf(request).some((part) => part.type === "image_url"))
    expect(withImage).toHaveLength(scans.length)
    for (const request of withImage) expect(partsOf(request)[0]?.text).toContain(LOW_TRUST_NOTE)

    const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as { ocrPages: unknown[] }
    expect(state.ocrPages.length).toBeGreaterThanOrEqual(scans.length)
  })

  it("never asks about a drawing, an empty page or a repeated line", async () => {
    await run(deterministicModel)
    const texts = requests.map((request) => partsOf(request)[0]?.text ?? "")
    expect(texts.some((text) => text.includes("A-100 2400"))).toBe(false)
    expect(texts.some((text) => text.includes("Customer: Example Ltd"))).toBe(false)
    expect(texts.some((text) => text.includes("Puslapio poraštė"))).toBe(false)
  })

  it("fails when a drawing is read: the lines nobody expected cost precision", async () => {
    const reading = modelLineExtractor({ gateway, model: "eval-model", excludedPageClasses: [] })
    platform.ai.respond(deterministicModel)
    const results: CaseResult[] = []
    for (const golden of cases) {
      results.push(scoreCase(golden, await extractCase(reading, golden, await pagesThroughPlatform(golden))))
    }
    const metrics = aggregate(results)
    expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
    expect(results.flatMap((result) => result.skippedWrongly).join(" ")).toContain("not skipped")
  })

  it("fails when the model's quotes are not on the page: the verification is measured, not assumed", async () => {
    // every digit of every quote shifted by one: plausible-looking lines whose words are not on the page
    const { metrics } = await run((request) => {
      const reply = deterministicModel(request)
      const body = JSON.parse(reply.content ?? "{}") as { lines?: Array<{ quote: string }> }
      for (const line of body.lines ?? [])
        line.quote = line.quote.replace(/\d/g, (digit) => String((Number(digit) + 1) % 10))
      return { content: JSON.stringify(body) }
    })
    expect(metrics.verification).toBeLessThan(thresholds.offline.verification)
    expect(failuresOf(metrics, thresholds.offline).join(" ")).toContain("verification")
  })

  it("fails when the model finds nothing: the recall is measured", async () => {
    const { metrics } = await run(() => ({ content: JSON.stringify({ lines: [] }) }))
    expect(metrics.recall).toBe(0)
    expect(failuresOf(metrics, thresholds.offline).join(" ")).toContain("recall")
  })

  it("fails when the model invents lines: the precision is measured", async () => {
    const { metrics } = await run((request) => {
      const reply = deterministicModel(request)
      const body = JSON.parse(reply.content ?? "{}") as {
        lines?: Array<{ label: string; value: string; quote: string }>
      }
      body.lines = [...(body.lines ?? []), { label: "Invented", value: "1", quote: "Invented: 1" }]
      return { content: JSON.stringify(body) }
    })
    expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
  })
})

describe.skipIf(!live)("live, against the platform's AI gateway", () => {
  it("scores line recall, precision and quote verification above the live thresholds", async () => {
    const url = process.env.CONUSAI_API_URL
    const token = process.env.CONUSAI_API_TOKEN
    if (!url || !token) throw new Error("EVAL_LIVE=1 needs CONUSAI_API_URL and CONUSAI_API_TOKEN")
    const model = aiModelOf()
    const { results, metrics } = await evaluate(
      createModelGateway({ CONUSAI_API_URL: url, CONUSAI_API_TOKEN: token }, { model }),
      model,
    )
    console.log(`model: ${model}\n${report(results, metrics)}`)
    expect(failuresOf(metrics, thresholds.live)).toEqual([])
  }, 600_000)
})