ConusAI vendor v0.1.0-dev.20261027+9b1131fb

modules/documents/contract/constants.ts

@@ -0,0 +1,54 @@
+// The module's closed vocabularies, in a file with no import: the schemas build their enums from
+// these, and a browser bundle can take them without loading the data layer the schemas live beside.
+
+export const DOCUMENT_STATUSES = ["processing", "extracted", "failed"] as const
+export type DocumentStatus = (typeof DOCUMENT_STATUSES)[number]
+
+/** Why a document failed, as a code. A reason carrying document text would be a customer's value in a log. */
+export const FAILURE_CODES = [
+  "file_missing",
+  "file_changed",
+  "ocr_failed",
+  "extraction_refused",
+  "extraction_truncated",
+  "extraction_invalid",
+] as const
+export type FailureCode = (typeof FAILURE_CODES)[number]
+
+/**
+ * Whether the words an extractor quoted were found on the page they cite. A line whose quote is not
+ * there is not dropped and not trusted: it is `needs_review`, and says so to the reviewer.
+ */
+export const LINE_VERIFICATIONS = ["verified", "needs_review"] as const
+export type LineVerification = (typeof LINE_VERIFICATIONS)[number]
+
+/**
+ * What kind of page the reader found, from its own signals (ocr-service OCR3.15): `text` a born-digital
+ * text layer, `table`, `drawing` (a technical sheet: a few labels or hundreds of short fragments),
+ * `scan` (a recognised text page) and `empty`.
+ */
+export const PAGE_CLASSES = ["text", "table", "drawing", "scan", "empty"] as const
+export type PageClass = (typeof PAGE_CLASSES)[number]
+
+/** The page classes named, or the first name that is not one: a deployment's list is checked once, where it is read. */
+export function pageClassesOf(names: readonly string[]): PageClass[] {
+  return names.map((name) => {
+    if (!(PAGE_CLASSES as readonly string[]).includes(name)) {
+      throw new Error(`"${name}" is not a page class: expected one of ${PAGE_CLASSES.join(", ")}`)
+    }
+    return name as PageClass
+  })
+}
+
+/** The page classes the extraction leaves out unless the deployment says otherwise: nothing on them is a specification. */
+export const DEFAULT_EXCLUDED_PAGE_CLASSES: readonly PageClass[] = ["drawing", "empty"]
+
+/**
+ * Why a reviewer should look at a line besides an unfound quote. `scanned_quote`: the words come from
+ * the image of a scanned page, and could not be found in its recognised text, so only the page image can vouch for them.
+ */
+export const LINE_NOTES = ["scanned_quote"] as const
+export type LineNote = (typeof LINE_NOTES)[number]
+
+export const LINE_STATUSES = ["pending", "approved", "rejected"] as const
+export type LineStatus = (typeof LINE_STATUSES)[number]

modules/documents/contract/entities.ts

@@ -0,0 +1,64 @@
+// The tables of the documents module (spec §16). Two, both project-owned.
+//
+// `documents_document` is one uploaded file's processing state. `documents_line` holds only what a
+// person has APPROVED: an extracted line lives as a proposal (kit-proposals) until somebody
+// decides it, and the proposal's writer inserts the line in the approving transaction. That is how
+// "nothing is exported without approval" is a property of the schema and not of a filter: the
+// export reads this table, and nothing but an approval writes to it.
+
+import { defineEntity, idSchema, instantSchema } from "@conusai-kit/data"
+import { z } from "zod"
+import { DOCUMENT_STATUSES, FAILURE_CODES, PAGE_CLASSES } from "./constants.js"
+
+export const documents = defineEntity({
+  name: "document",
+  table: "documents_document",
+  tenancy: "project",
+  fields: {
+    /** The row of the files module this document was made from: an opaque id, never a foreign key (spec §4.4). */
+    fileId: idSchema,
+    filename: z.string().min(1).max(255),
+    status: z.enum(DOCUMENT_STATUSES),
+    /** The platform's OCR job id, recorded as soon as the job is accepted so a retry resumes it. */
+    ocrJobId: z.number().int().nonnegative().nullable(),
+    pageCount: z.number().int().nonnegative().nullable(),
+    /** How many lines the extraction proposed; the proposals themselves are the reviewer's queue. */
+    proposedCount: z.number().int().nonnegative().nullable(),
+    failureCode: z.enum(FAILURE_CODES).nullable(),
+    /** The pages the extraction left out, with the class that excluded each, so a reviewer sees what was not read. Null until extracted. */
+    skippedPages: z
+      .array(z.object({ page: z.number().int().min(1), pageClass: z.enum(PAGE_CLASSES) }).strict())
+      .nullable(),
+    finishedAt: instantSchema.nullable(),
+  },
+  // One document per file: the job that creates it is retried, and a second row would be a second queue.
+  unique: [["fileId"]],
+  indexes: [["status"]],
+})
+
+export const lines = defineEntity({
+  name: "line",
+  table: "documents_line",
+  tenancy: "project",
+  fields: {
+    documentId: idSchema,
+    /** The approved proposal this line came from: its provenance, and the reason a line is written once. */
+    proposalId: idSchema,
+    label: z.string().min(1).max(200),
+    value: z.string().min(1).max(2_000),
+    /** 1-based page of the source document. */
+    page: z.number().int().min(1),
+    /** Reading order within the document: the export's order, which a creation time cannot give. */
+    position: z.number().int().min(0),
+    /** The words the value was read from, verbatim. */
+    quote: z.string().min(1).max(2_000),
+    /** True when a reviewer changed the label or value before it was approved. */
+    edited: z.boolean(),
+    approvedBy: z.string().min(1).max(128),
+  },
+  unique: [["proposalId"]],
+  indexes: [["documentId"]],
+})
+
+/** Every table this module owns, in the order a migration creates them. */
+export const DOCUMENTS_ENTITIES = Object.freeze([documents, lines])

modules/documents/contract/index.ts

@@ -0,0 +1,45 @@
+// The public API of the documents module: what a page, another module and the OpenAPI document
+// may import. Nothing here reaches core/, adapters/ or a driver.
+
+export {
+  DEFAULT_EXCLUDED_PAGE_CLASSES,
+  DOCUMENT_STATUSES,
+  type DocumentStatus,
+  FAILURE_CODES,
+  type FailureCode,
+  LINE_NOTES,
+  LINE_STATUSES,
+  LINE_VERIFICATIONS,
+  type LineNote,
+  type LineStatus,
+  type LineVerification,
+  PAGE_CLASSES,
+  type PageClass,
+  pageClassesOf,
+} from "./constants.js"
+export { DOCUMENTS_ENTITIES, documents, lines } from "./entities.js"
+export {
+  EXPORT_CREATED,
+  LINE_EDITED,
+  LINE_TARGET_KEY,
+  type LineChange,
+  type LineView,
+  lineChangeSchema,
+  lineViewSchema,
+  MAX_MODEL_LINES,
+  type ModelLines,
+  modelLineSchema,
+  modelLinesSchema,
+  OCR_JOB,
+  PROCESSOR_KEY,
+  READABLE_CONTENT_TYPES,
+} from "./lines.js"
+export {
+  type DocumentView,
+  documentGet,
+  documentList,
+  documentViewSchema,
+  exportCreate,
+  lineEdit,
+} from "./procedures.js"
+export { REVIEW_PROCEDURES } from "./review.js"

modules/documents/contract/lines.ts

@@ -0,0 +1,84 @@
+// What an extracted line is, as the proposal that carries it and as the reviewer sees it.
+
+import { z } from "zod"
+import { LINE_NOTES, LINE_STATUSES, LINE_VERIFICATIONS } from "./constants.js"
+
+/** The proposal target this module contributes to `proposals.targets`: the writer of approved lines. */
+export const LINE_TARGET_KEY = "documents.line"
+
+/** The `files.processors` key and the job it names. */
+export const PROCESSOR_KEY = "documents.ocr"
+export const OCR_JOB = "documents.ocr"
+
+/** The audit events of this module, declared in its manifest. */
+export const EXPORT_CREATED = "documents.export_created"
+export const LINE_EDITED = "documents.line_edited"
+
+/** The content types the module reads. Spreadsheets are the customer's own extraction, not the template's. */
+export const READABLE_CONTENT_TYPES = ["application/pdf", "image/png", "image/jpeg"] as const
+
+/**
+ * The change a proposal carries and approval writes. `lineKey` is derived from the document and the
+ * line's position, so an extraction retried after a crash adds nothing a person already decided.
+ */
+export const lineChangeSchema = z
+  .object({
+    lineKey: z.string().regex(/^[0-9a-f]{64}$/, "a lower-case hex SHA-256"),
+    label: z.string().min(1).max(200),
+    value: z.string().min(1).max(2_000),
+    page: z.number().int().min(1),
+    /** Where the line sits in the document's reading order: what a review and an export sort by. */
+    position: z.number().int().min(0),
+    quote: z.string().min(1).max(2_000),
+    /** The reader's confidence in the words the line was read from, 0 to 1; a reviewer sorts by it. */
+    confidence: z.number().min(0).max(1),
+    /** `needs_review` when the quote was not found on the page it cites (see `LINE_VERIFICATIONS`). */
+    verification: z.enum(LINE_VERIFICATIONS),
+    /** Why a reviewer should look at the line besides `verification`; absent when there is nothing to add. */
+    note: z.enum(LINE_NOTES).optional(),
+    /** True once a reviewer has corrected the label or the value (the proposals module records who and when). */
+    edited: z.boolean(),
+  })
+  .strict()
+
+export type LineChange = z.output<typeof lineChangeSchema>
+
+/**
+ * What a model is asked to return for one page: the line schema above, cut to the fields a model can
+ * know. The page is the question's, not the answer's (a model that cites a page may cite the wrong
+ * one), the position and the key are the code's, and a confidence is measured from the reader, not
+ * asked of the model. The JSON Schema sent to the gateway is derived from this.
+ */
+export const modelLineSchema = lineChangeSchema.pick({ label: true, value: true, quote: true }).strict()
+
+export const MAX_MODEL_LINES = 200
+
+export const modelLinesSchema = z.object({ lines: z.array(modelLineSchema).max(MAX_MODEL_LINES) }).strict()
+
+export type ModelLines = z.output<typeof modelLinesSchema>
+
+/** One line as the review screen shows it: the change, where it was read, and what was decided. */
+export const lineViewSchema = z.object({
+  /** The proposal's id: it is what approve, reject and edit are called with. */
+  lineId: z.string(),
+  /** The proposal version the reviewer saw; approval pins to it to prevent approving a later edit. */
+  expectedVersion: z.number().int().min(1),
+  status: z.enum(LINE_STATUSES),
+  label: z.string(),
+  value: z.string(),
+  page: z.number().int(),
+  quote: z.string(),
+  box: z.object({ x: z.number(), y: z.number(), width: z.number(), height: z.number() }).nullable(),
+  confidence: z.number(),
+  verification: z.enum(LINE_VERIFICATIONS),
+  note: z.enum(LINE_NOTES).nullable(),
+  /** The reader's stable id of the block the quote sits in (`p{page}-b{n}`), when the reader gave one. */
+  blockId: z.string().nullable(),
+  edited: z.boolean(),
+  /** The model and version that produced the line: the disclosure Art. 50 asks for sits beside it. */
+  model: z.string(),
+  promptVersion: z.string(),
+  decidedAt: z.string().nullable(),
+})
+
+export type LineView = z.output<typeof lineViewSchema>

modules/documents/contract/procedures.ts

@@ -0,0 +1,80 @@
+// The procedure contracts of the documents module. This file is what another module, a browser and
+// the OpenAPI document may import: schemas and names, no handler and no port.
+//
+// Approving and rejecting a line are not here. They are the proposals module's own procedures
+// (`proposals.proposal.approve`, `.reject`, `.approveMany`), which this module's writer is called
+// from, so a review decision has one implementation, one permission and one audit event whichever
+// screen it was made on (decision AT-D22).
+
+import { procedure } from "@conusai-kit/core"
+import { z } from "zod"
+import { DOCUMENT_STATUSES, FAILURE_CODES, PAGE_CLASSES } from "./constants.js"
+import { EXPORT_CREATED, LINE_EDITED, lineViewSchema } from "./lines.js"
+
+const inProject = { org: z.string(), workspace: z.string(), project: z.string() }
+
+/** A file with no document row yet has been uploaded and not picked up by the job: `queued`. */
+export const documentViewSchema = z.object({
+  fileId: z.string(),
+  filename: z.string(),
+  status: z.enum(["queued", ...DOCUMENT_STATUSES]),
+  failureCode: z.enum(FAILURE_CODES).nullable(),
+  pageCount: z.number().int().nullable(),
+  /** The pages the extraction left out, with the class that excluded each: a reviewer sees what was not read. */
+  skippedPages: z.array(z.object({ page: z.number().int().min(1), pageClass: z.enum(PAGE_CLASSES) })),
+  /** Lines waiting for a decision, and lines approved: the two numbers a reviewer scans a list for. */
+  pending: z.number().int(),
+  approved: z.number().int(),
+  uploadedAt: z.string(),
+})
+
+export type DocumentView = z.output<typeof documentViewSchema>
+
+export const documentList = procedure("documents.document.list")
+  .kind("read")
+  .scope("project")
+  .permission("documents.document.read")
+  .input(z.object({ ...inProject }))
+  .output(z.object({ documents: z.array(documentViewSchema) }))
+  .contract()
+
+export const documentGet = procedure("documents.document.get")
+  .kind("read")
+  .scope("project")
+  .permission("documents.document.read")
+  .input(z.object({ ...inProject, fileId: z.string() }))
+  .output(z.object({ document: documentViewSchema, lines: z.array(lineViewSchema) }))
+  .contract()
+
+export const lineEdit = procedure("documents.line.edit")
+  .kind("write")
+  .scope("project")
+  .permission("documents.line.edit")
+  .input(
+    z.object({
+      ...inProject,
+      lineId: z.string(),
+      label: z.string().min(1).max(200),
+      value: z.string().min(1).max(2_000),
+    }),
+  )
+  .output(lineViewSchema)
+  .audit(LINE_EDITED)
+  .contract()
+
+export const exportCreate = procedure("documents.export.create")
+  .kind("effect")
+  .scope("project")
+  .permission("documents.export.create")
+  .input(z.object({ ...inProject, fileId: z.string() }))
+  .output(
+    z.object({
+      filename: z.string(),
+      contentType: z.string(),
+      /** The workbook, standard base64: a procedure's answer has to survive JSON on every transport. */
+      body: z.string(),
+      lineCount: z.number().int(),
+    }),
+  )
+  .audit(EXPORT_CREATED)
+  .contract()

modules/documents/contract/review.ts

@@ -0,0 +1,13 @@
+// The names of the kernel's review procedures, for a page that calls them. A file of its own and
+// with no import, because a browser bundle takes it: the rest of the contract pulls in the data
+// layer, which a client component has no business loading.
+//
+// Approving and rejecting a line are the proposals module's procedures, not this module's: a
+// review decision has one implementation, one permission and one audit event whichever screen it
+// was made on (decision AT-D22).
+
+export const REVIEW_PROCEDURES = {
+  approve: "proposals.proposal.approve",
+  approveMany: "proposals.proposal.approveMany",
+  reject: "proposals.proposal.reject",
+} as const