ConusAI vendor v0.1.0-dev.20261027+9b1131fb

test/documents-flow.test.ts

@@ -0,0 +1,415 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The document-review flow end to end on the fake platform (plan item AT2.3): an uploaded file is
+// read by an OCR job, becomes proposals, a person approves, edits or rejects each line, and only
+// approved lines are exported. Every step runs through the kernel's caller, the files module's
+// processor point, the proposals module's writer and the outbox relay's behaviour, so what is proven
+// is the wiring and not a unit.
+
+import { readFileSync } from "node:fs"
+import { type LineView, lines as linesTable, REVIEW_PROCEDURES, type ReadDocument } from "@app/modules/documents"
+import { AppError } from "@conusai-kit/core"
+import { uploadViaProcedures } from "@conusai-kit/testing"
+import { afterEach, describe, expect, it } from "vitest"
+import { sheetRows } from "../modules/documents/__tests__/support/unzip.js"
+import {
+  bytesOf,
+  closeApps,
+  type DocumentsApp,
+  makeDocumentsApp,
+  type SeededProject,
+  uploadAndProcess,
+} from "./support/documents-app.js"
+
+const document = JSON.parse(
+  readFileSync(new URL("../modules/documents/__tests__/fixtures/ocr-document.json", import.meta.url), "utf8"),
+) as ReadDocument & Record<string, unknown>
+
+afterEach(closeApps)
+
+interface Detail {
+  document: {
+    status: string
+    failureCode: string | null
+    pending: number
+    approved: number
+    pageCount: number | null
+    skippedPages: unknown[]
+  }
+  lines: LineView[]
+}
+
+async function fresh(): Promise<{ app: DocumentsApp; project: SeededProject }> {
+  const app = await makeDocumentsApp({ document })
+  return { app, project: await app.seedProject("acme", "owner@acme.test") }
+}
+
+const detail = async (project: SeededProject, fileId: string): Promise<Detail> =>
+  (await project.call.callByName("documents.document.get", { ...project.slugs, fileId })) as Detail
+
+const decide = (project: SeededProject, procedure: string, input: Record<string, unknown>) =>
+  project.call.callByName(procedure, { ...project.slugs, ...input })
+
+const exportRows = async (project: SeededProject, fileId: string) => {
+  const made = (await decide(project, "documents.export.create", { fileId })) as {
+    body: string
+    lineCount: number
+    filename: string
+  }
+  return { ...made, rows: sheetRows(Buffer.from(made.body, "base64")) }
+}
+
+const refusal = async (promise: Promise<unknown>): Promise<AppError> => {
+  try {
+    await promise
+  } catch (error) {
+    if (error instanceof AppError) return error
+    throw error
+  }
+  throw new Error("expected a refusal")
+}
+
+describe("uploading a document", () => {
+  it("answers before OCR runs: the request enqueues, and the worker reads", async () => {
+    const { app, project } = await fresh()
+    await uploadViaProcedures(project.call, project.slugs, {
+      filename: "report.pdf",
+      contentType: "application/pdf",
+      bytes: bytesOf("%PDF-1.4 fixture"),
+    })
+    // The upload has returned and the reader has not been asked anything.
+    expect(app.ocr.submitted).toHaveLength(0)
+    expect(app.queue.of("documents.ocr")).toHaveLength(0)
+    // The stored file's event is what enqueues the job, once.
+    await app.deliver()
+    expect(app.queue.of("documents.ocr")).toHaveLength(1)
+    expect(app.ocr.submitted).toHaveLength(0)
+    await app.runJobs()
+    expect(app.ocr.submitted).toHaveLength(1)
+  })
+
+  it("is not read by this module when it is not a document type it reads", async () => {
+    const { app, project } = await fresh()
+    await uploadViaProcedures(project.call, project.slugs, {
+      filename: "notes.csv",
+      contentType: "text/csv",
+      bytes: bytesOf("a,b"),
+    })
+    await app.deliver()
+    expect(app.queue.of("documents.ocr")).toHaveLength(0)
+  })
+
+  it("lists an upload nobody has read yet as queued", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadViaProcedures(project.call, project.slugs, {
+      filename: "report.pdf",
+      contentType: "application/pdf",
+      bytes: bytesOf("%PDF-1.4"),
+    })
+    const { documents } = (await decide(project, "documents.document.list", {})) as {
+      documents: Array<{ fileId: string; status: string }>
+    }
+    expect(documents).toEqual([expect.objectContaining({ fileId: file.fileId, status: "queued" })])
+    expect(app.ocr.submitted).toHaveLength(0)
+  })
+})
+
+describe("the extraction", () => {
+  it("turns the fixture document into one pending line per labelled line, each with its page and verbatim quote", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const { document: view, lines } = await detail(project, file.fileId)
+
+    // the fixture's third page is a drawing: left out, and said so on the document
+    expect(view).toMatchObject({
+      status: "extracted",
+      pending: 4,
+      approved: 0,
+      pageCount: 3,
+      skippedPages: [{ page: 3, pageClass: "drawing" }],
+    })
+    expect(lines.map((line) => [line.page, line.label, line.value, line.status])).toEqual([
+      [1, "Reference", "QR-2026-04", "pending"],
+      [1, "Issued on", "2026-04-01", "pending"],
+      [2, "Total amount", "1 250,00 EUR", "pending"],
+      [2, "Signed by", "A. Example", "pending"],
+    ])
+    for (const line of lines) {
+      expect(document.pages[line.page - 1]?.text).toContain(line.quote)
+      expect(line.box).not.toBeNull()
+      // The line says which engine and extractor produced it: the Art. 50 disclosure's subject.
+      expect(line.model).toContain("fixture-engine")
+      expect(line.promptVersion).toBe("extract-lines.v2")
+    }
+  })
+
+  it("submits the document once, with a key that names the file and its content", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    expect(app.ocr.submitted).toHaveLength(1)
+    expect(app.ocr.submitted[0]?.idempotencyKey).toMatch(
+      new RegExp(`^documents\\.ocr\\.${file.fileId}\\.[0-9a-f]{64}$`),
+    )
+  })
+
+  it("writes nothing the reviewer has not approved", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    expect(
+      await app.data
+        .forScope({ level: "project", actor: project.owner, ...scopeIds(project) })
+        .of(linesTable)
+        .count(),
+    ).toBe(0)
+    // and so an export has nothing to say yet
+    const error = await refusal(decide(project, "documents.export.create", { fileId: file.fileId }))
+    expect(error.status).toBe(409)
+  })
+})
+
+describe("retrying the job", () => {
+  it("a job run twice adds no proposal and submits no second document", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const job = app.queue.jobs.find((entry) => entry.job === "documents.ocr")
+    expect(job).toBeDefined()
+    // The queue delivers the same job again (a lapsed claim): run its handler once more.
+    const again = app.queue.jobs.length
+    await app.queue.enqueue(job?.scope as never, "documents.ocr", job?.payload as never)
+    expect(app.queue.jobs).toHaveLength(again + 1)
+    await app.runJobs()
+    expect(app.ocr.submitted).toHaveLength(1)
+    expect((await detail(project, file.fileId)).lines).toHaveLength(4)
+  })
+
+  it("a transient failure while waiting resumes the stored OCR job instead of submitting another", async () => {
+    const { app, project } = await fresh()
+    app.ocr.failWaitOnce(new Error("the platform is unreachable"))
+    const file = await uploadViaProcedures(project.call, project.slugs, {
+      filename: "report.pdf",
+      contentType: "application/pdf",
+      bytes: bytesOf("%PDF-1.4"),
+    })
+    await app.deliver()
+    await expect(app.runJobs()).rejects.toThrow("unreachable")
+    expect((await detail(project, file.fileId)).document.status).toBe("processing")
+
+    // pg-boss retries a job that throws: the same payload, the same scope.
+    const job = app.queue.jobs.find((entry) => entry.job === "documents.ocr")
+    await app.queue.enqueue(job?.scope as never, "documents.ocr", job?.payload as never, { key: "retry" })
+    await app.runJobs()
+    expect(app.ocr.submitted).toHaveLength(1)
+    expect(app.ocr.waited).toHaveLength(2)
+    expect((await detail(project, file.fileId)).document).toMatchObject({ status: "extracted", pending: 4 })
+  })
+
+  it("marks the document failed, with a code and no proposals, when the platform says the job failed", async () => {
+    const { app, project } = await fresh()
+    app.platform.ocr.failJobs(true)
+    const file = await uploadAndProcess(app, project)
+    const { document: view, lines } = await detail(project, file.fileId)
+    expect(view).toMatchObject({ status: "failed", failureCode: "ocr_failed" })
+    expect(lines).toEqual([])
+  })
+
+  it("marks the document failed when the stored bytes are not the ones that were uploaded", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadViaProcedures(project.call, project.slugs, {
+      filename: "report.pdf",
+      contentType: "application/pdf",
+      bytes: bytesOf("%PDF-1.4 original"),
+    })
+    for (const [key, object] of app.store.objects)
+      app.store.objects.set(key, { ...object, bytes: new TextEncoder().encode("tampered") })
+    await app.deliver()
+    await app.runJobs()
+    expect((await detail(project, file.fileId)).document).toMatchObject({
+      status: "failed",
+      failureCode: "file_changed",
+    })
+    expect(app.ocr.submitted).toHaveLength(0)
+  })
+})
+
+describe("the review", () => {
+  it("approving a line writes the record and one audit event; the other lines stay pending", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [first] = (await detail(project, file.fileId)).lines
+    await decide(project, REVIEW_PROCEDURES.approve, {
+      proposalId: first?.lineId,
+      expectedVersion: first?.expectedVersion,
+    })
+    await app.deliver()
+
+    const after = await detail(project, file.fileId)
+    expect(after.document).toMatchObject({ pending: 3, approved: 1 })
+    expect(after.lines.find((line) => line.lineId === first?.lineId)?.status).toBe("approved")
+    const approvals = app.audit.events.filter((event) => event.action === "proposals.proposal_approved")
+    expect(approvals).toHaveLength(1)
+    // ids and hashes only: no label, value or quote in an audit event
+    expect(JSON.stringify(approvals[0])).not.toContain("QR-2026-04")
+  })
+
+  it("rejecting a line changes no record, and the line stays rejected with its audit event", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [first] = (await detail(project, file.fileId)).lines
+    await decide(project, REVIEW_PROCEDURES.reject, { proposalId: first?.lineId, reason: "not a field" })
+    await app.deliver()
+
+    const after = await detail(project, file.fileId)
+    expect(after.lines.find((line) => line.lineId === first?.lineId)?.status).toBe("rejected")
+    expect(after.document).toMatchObject({ pending: 3, approved: 0 })
+    expect(app.audit.events.filter((event) => event.action === "proposals.proposal_rejected")).toHaveLength(1)
+    // nothing was written to the approved table: the export has nothing to say
+    expect((await refusal(decide(project, "documents.export.create", { fileId: file.fileId }))).status).toBe(409)
+  })
+
+  it("an edit replaces the line with the reviewer's words, which are then approved like any other", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [, second] = (await detail(project, file.fileId)).lines
+    const edited = (await decide(project, "documents.line.edit", {
+      lineId: second?.lineId,
+      label: "Issue date",
+      value: "2026-04-02",
+    })) as LineView
+    // the same line, with the reviewer's words in it
+    expect(edited).toMatchObject({
+      lineId: second?.lineId,
+      label: "Issue date",
+      value: "2026-04-02",
+      edited: true,
+      status: "pending",
+    })
+    // the quote still names the words the machine read, not the reviewer's
+    expect(edited.quote).toBe(second?.quote)
+
+    const listed = (await detail(project, file.fileId)).lines
+    expect(listed).toHaveLength(4)
+    expect(listed.find((line) => line.lineId === second?.lineId)).toMatchObject({ label: "Issue date", edited: true })
+    await app.deliver()
+    // one audit event for the edit, with no word of it
+    const audited = app.audit.events.filter((event) => event.action === "proposals.proposal_edited")
+    expect(audited).toHaveLength(1)
+    expect(JSON.stringify(audited[0])).not.toContain("Issue date")
+
+    await decide(project, REVIEW_PROCEDURES.approve, {
+      proposalId: edited.lineId,
+      expectedVersion: edited.expectedVersion,
+    })
+    const { rows } = await exportRows(project, file.fileId)
+    expect(rows).toEqual([
+      ["Label", "Value", "Page", "Source quote", "Edited by a reviewer"],
+      ["Issue date", "2026-04-02", 1, "Issued on: 2026-04-01", "yes"],
+    ])
+  })
+
+  it("a line that has been decided cannot be edited", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [first] = (await detail(project, file.fileId)).lines
+    await decide(project, REVIEW_PROCEDURES.reject, { proposalId: first?.lineId })
+    const error = await refusal(
+      decide(project, "documents.line.edit", { lineId: first?.lineId, label: "x", value: "y" }),
+    )
+    expect(error.status).toBe(409)
+  })
+
+  it("approves every pending line in one decision", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const lines = (await detail(project, file.fileId)).lines
+    const ids = lines.map((line) => line.lineId)
+    const expectations = Object.fromEntries(
+      ids.map((id, index) => [id, { expectedVersion: lines[index]?.expectedVersion }]),
+    )
+    await decide(project, REVIEW_PROCEDURES.approveMany, { proposalIds: ids, expectations })
+    expect((await detail(project, file.fileId)).document).toMatchObject({ pending: 0, approved: 4 })
+  })
+})
+
+describe("the export", () => {
+  it("contains the approved lines and only those, as an Excel workbook", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project, "quarterly.report.pdf")
+    const lines = (await detail(project, file.fileId)).lines
+    await decide(project, REVIEW_PROCEDURES.approve, {
+      proposalId: lines[2]?.lineId,
+      expectedVersion: lines[2]?.expectedVersion,
+    })
+    await decide(project, REVIEW_PROCEDURES.approve, {
+      proposalId: lines[0]?.lineId,
+      expectedVersion: lines[0]?.expectedVersion,
+    })
+    await decide(project, REVIEW_PROCEDURES.reject, { proposalId: lines[1]?.lineId })
+
+    const made = await exportRows(project, file.fileId)
+    expect(made.filename).toBe("quarterly.report-approved.xlsx")
+    expect(made.lineCount).toBe(2)
+    expect(made.rows.slice(1)).toEqual([
+      ["Reference", "QR-2026-04", 1, "Reference: QR-2026-04", "no"],
+      ["Total amount", "1 250,00 EUR", 2, "Total amount: 1 250,00 EUR", "no"],
+    ])
+  })
+
+  it("writes one audit event naming no content", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [first] = (await detail(project, file.fileId)).lines
+    await decide(project, REVIEW_PROCEDURES.approve, {
+      proposalId: first?.lineId,
+      expectedVersion: first?.expectedVersion,
+    })
+    await decide(project, "documents.export.create", { fileId: file.fileId })
+    // The transport records a procedure's declared audit event (apps/web router); here the
+    // contract's own declaration is the proof, since no transport runs.
+    expect(app.registry.procedures.get("documents.export.create")?.def.audit).toBe("documents.export_created")
+    expect(app.registry.procedures.get("documents.line.edit")?.def.audit).toBe("documents.line_edited")
+  })
+})
+
+describe("who may do what", () => {
+  it("lets a viewer read a document and refuses them (404, as a stranger) an edit, an approval and an export", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [first] = (await detail(project, file.fileId)).lines
+    const viewer = await app.addMember(project, "viewer@acme.test", "viewer")
+
+    const read = (await viewer.call.callByName("documents.document.get", {
+      ...project.slugs,
+      fileId: file.fileId,
+    })) as Detail
+    expect(read.lines).toHaveLength(4)
+    for (const [name, input] of [
+      ["documents.line.edit", { lineId: first?.lineId, label: "x", value: "y" }],
+      [REVIEW_PROCEDURES.approve, { proposalId: first?.lineId, expectedVersion: first?.expectedVersion }],
+      ["documents.export.create", { fileId: file.fileId }],
+    ] as const) {
+      const error = await refusal(viewer.call.callByName(name, { ...project.slugs, ...input }))
+      // The kernel answers a member who lacks the permission exactly as it answers a stranger
+      // (spec §3.3): 404, so a refusal does not say which procedures exist for whom.
+      expect(error.status, name).toBe(404)
+    }
+    expect((await detail(project, file.fileId)).document.approved).toBe(0)
+  })
+
+  it("lets a member review: the permission of the line's writer is the member role's", async () => {
+    const { app, project } = await fresh()
+    const file = await uploadAndProcess(app, project)
+    const [first] = (await detail(project, file.fileId)).lines
+    const member = await app.addMember(project, "member@acme.test", "member")
+    await member.call.callByName(REVIEW_PROCEDURES.approve, {
+      ...project.slugs,
+      proposalId: first?.lineId,
+      expectedVersion: first?.expectedVersion,
+    })
+    expect((await detail(project, file.fileId)).document.approved).toBe(1)
+  })
+})
+
+const scopeIds = (project: SeededProject) => ({
+  orgId: project.orgId,
+  workspaceId: project.workspaceId,
+  projectId: project.projectId,
+})