This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The whole app of this template, wired the way apps/web and apps/worker wire it, on the in-memory
|
||||
// data source and the fakes of fakes.ts: the registry of every module, one caller per actor, the
|
||||
// outbox relay's behaviour and the worker's job runner.
|
||||
//
|
||||
// `deliver()` stands in for the outbox relay (kit-jobs-pgboss): it drains the outbox once, runs each
|
||||
// record's subscribers in a child container with the record's organization and a system actor, and
|
||||
// marks it published -- exactly once per subscriber, never more forgiving than the real relay.
|
||||
// `runJobs()` stands in for the worker: it runs every queued job in a child container carrying the
|
||||
// scope the job was enqueued in, with a system actor. What neither does is retry, so a test of a
|
||||
// retry runs the handler again itself.
|
||||
|
||||
import {
|
||||
DOCUMENT_EXCLUDED_PAGE_CLASSES,
|
||||
DOCUMENT_MODEL,
|
||||
DOCUMENT_OCR,
|
||||
DOCUMENTS_SCOPE,
|
||||
MODEL_GATEWAY,
|
||||
type PageClass,
|
||||
} from "@app/modules/documents"
|
||||
import { type Actor, buildRegistry, Container, createCaller, type TransportCaller } from "@conusai-kit/core"
|
||||
import { systemActor } from "@conusai-kit/core/worker"
|
||||
import {
|
||||
createMemoryDataSource,
|
||||
DATA_SOURCE,
|
||||
entitiesOf,
|
||||
JOB_QUEUE,
|
||||
OBJECT_STORE,
|
||||
SYSTEM_DATA,
|
||||
} from "@conusai-kit/data"
|
||||
import { FILES_AUDIT, FILES_REGISTRY, FILES_SCOPE, FILES_USAGE } from "@conusai-kit/files"
|
||||
import { createModelGateway } from "@conusai-kit/platform"
|
||||
import { PROPOSALS_AUDIT, PROPOSALS_REGISTRY, PROPOSALS_SCOPE } from "@conusai-kit/proposals"
|
||||
import { createTenancyHooks, TENANCY, users } from "@conusai-kit/tenancy"
|
||||
import {
|
||||
fakeJobQueue,
|
||||
fakeObjectStore,
|
||||
recordingAudit,
|
||||
recordingUsage,
|
||||
uploadViaProcedures,
|
||||
} from "@conusai-kit/testing"
|
||||
import type { FakeChatResponder } from "@conusai-kit/testing/fake-platform"
|
||||
import { appModules } from "../../modules/index.js"
|
||||
import { deterministicModel } from "./fake-model.js"
|
||||
import { type FakePlatform, startOcr } from "./ocr.js"
|
||||
|
||||
const started: FakePlatform[] = []
|
||||
|
||||
/** Stops every fake platform the test file started: call it from `afterEach`. */
|
||||
export async function closeApps(): Promise<void> {
|
||||
await Promise.all(started.splice(0).map((platform) => platform.close()))
|
||||
}
|
||||
|
||||
export interface SeededProject {
|
||||
readonly owner: Actor
|
||||
readonly call: TransportCaller
|
||||
readonly orgSlug: string
|
||||
readonly workspaceSlug: string
|
||||
readonly projectSlug: string
|
||||
readonly orgId: string
|
||||
readonly workspaceId: string
|
||||
readonly projectId: string
|
||||
/** The three slugs a project-scoped procedure's input carries. */
|
||||
readonly slugs: { org: string; workspace: string; project: string }
|
||||
}
|
||||
|
||||
/** `document` is what the fake platform's OCR reads every file as: a `conusai.ocr.document/v1`. */
|
||||
export async function makeDocumentsApp(options: {
|
||||
document: Record<string, unknown>
|
||||
/** What the model answers; by default the deterministic reader of fake-model.ts. */
|
||||
responder?: FakeChatResponder
|
||||
/** The page classes the deployment leaves out; by default the module's own (drawings and empty pages). */
|
||||
excludedPageClasses?: readonly PageClass[]
|
||||
}) {
|
||||
const registry = buildRegistry([...appModules])
|
||||
const entities = entitiesOf(registry, "postgres")
|
||||
const data = createMemoryDataSource({ entities })
|
||||
const store = fakeObjectStore()
|
||||
const queue = fakeJobQueue()
|
||||
const audit = recordingAudit()
|
||||
const usage = recordingUsage()
|
||||
const { platform, ocr } = await startOcr(options.document)
|
||||
started.push(platform)
|
||||
|
||||
const root = new Container()
|
||||
root.value(DATA_SOURCE, data)
|
||||
root.value(SYSTEM_DATA, data)
|
||||
root.value(OBJECT_STORE, store)
|
||||
root.value(JOB_QUEUE, queue)
|
||||
root.value(FILES_REGISTRY, registry)
|
||||
root.value(FILES_AUDIT, audit)
|
||||
root.value(FILES_USAGE, usage)
|
||||
root.value(PROPOSALS_REGISTRY, registry)
|
||||
root.value(PROPOSALS_AUDIT, audit)
|
||||
root.value(DOCUMENT_OCR, ocr)
|
||||
// The model is the real gateway client over HTTP to the fake platform, whose reply is scripted.
|
||||
const modelRequests: Array<Record<string, unknown>> = []
|
||||
const script = (responder: FakeChatResponder | undefined): void =>
|
||||
platform.ai.respond(
|
||||
responder &&
|
||||
((request) => {
|
||||
modelRequests.push(request)
|
||||
return responder(request)
|
||||
}),
|
||||
)
|
||||
script(options.responder ?? deterministicModel)
|
||||
root.value(
|
||||
MODEL_GATEWAY,
|
||||
createModelGateway({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "deployment-token" }),
|
||||
)
|
||||
root.value(DOCUMENT_MODEL, "fake-model")
|
||||
if (options.excludedPageClasses) root.value(DOCUMENT_EXCLUDED_PAGE_CLASSES, options.excludedPageClasses)
|
||||
|
||||
const caller = (actor: Actor): TransportCaller => {
|
||||
const tenancy = createTenancyHooks({ data, registry })
|
||||
const container = root.child()
|
||||
container.value(TENANCY, tenancy)
|
||||
return createCaller({ registry, container, tenancy }, { actor })
|
||||
}
|
||||
|
||||
const signUp = async (email: string): Promise<Actor> => {
|
||||
const user = await data.global(users).insert({
|
||||
issuer: "https://id.example.test",
|
||||
subject: email,
|
||||
email,
|
||||
displayName: email.split("@")[0] ?? email,
|
||||
locale: "en",
|
||||
})
|
||||
return { userId: user.id }
|
||||
}
|
||||
|
||||
const delivered: Array<{ name: string }> = []
|
||||
const deliver = async (): Promise<number> => {
|
||||
const pending = await data.outbox.pending(500)
|
||||
let ran = 0
|
||||
for (const record of pending) {
|
||||
const entry = registry.events[record.name]
|
||||
if (entry) {
|
||||
for (const subscriber of entry.subscribers) {
|
||||
const container = root.child()
|
||||
const scope = { level: "org" as const, actor: systemActor("app.relay"), orgId: record.orgId }
|
||||
container.value(FILES_SCOPE, scope)
|
||||
container.value(PROPOSALS_SCOPE, scope)
|
||||
await subscriber.handler(
|
||||
{ name: record.name, payload: record.payload },
|
||||
{ container, correlationId: record.id },
|
||||
)
|
||||
delivered.push({ name: record.name })
|
||||
ran += 1
|
||||
}
|
||||
}
|
||||
await data.outbox.markPublished([record.id])
|
||||
}
|
||||
return ran
|
||||
}
|
||||
|
||||
/** The jobs not yet run, in the order they were enqueued. */
|
||||
let consumed = 0
|
||||
const runJobs = async (): Promise<number> => {
|
||||
let ran = 0
|
||||
while (consumed < queue.jobs.length) {
|
||||
const job = queue.jobs[consumed] as (typeof queue.jobs)[number]
|
||||
consumed += 1
|
||||
const registered = registry.jobs[job.job]
|
||||
if (!registered) throw new Error(`no module declares the job ${job.job}`)
|
||||
const container = root.child()
|
||||
const tenancy = createTenancyHooks({ data, registry })
|
||||
container.value(TENANCY, tenancy)
|
||||
container.value(DOCUMENTS_SCOPE, { ...job.scope, actor: systemActor(`${job.job}`) })
|
||||
await registered.def.handler(registered.def.payload.parse(job.payload), {
|
||||
container,
|
||||
correlationId: job.id,
|
||||
attempt: 1,
|
||||
})
|
||||
ran += 1
|
||||
}
|
||||
return ran
|
||||
}
|
||||
|
||||
async function seedProject(slug: string, email: string): Promise<SeededProject> {
|
||||
const owner = await signUp(email)
|
||||
const call = caller(owner)
|
||||
const org = (await call.callByName("tenancy.org.create", { slug, name: slug })) as { orgId: string; slug: string }
|
||||
const workspace = (await call.callByName("tenancy.workspace.create", {
|
||||
org: slug,
|
||||
slug: `ws-${slug}`,
|
||||
name: `Workspace ${slug}`,
|
||||
visibility: "org",
|
||||
})) as { workspaceId: string; slug: string }
|
||||
const project = (await call.callByName("tenancy.project.create", {
|
||||
org: slug,
|
||||
workspace: workspace.slug,
|
||||
slug: `p-${slug}`,
|
||||
name: `Project ${slug}`,
|
||||
moduleKey: "documents",
|
||||
})) as { projectId: string; slug: string }
|
||||
return {
|
||||
owner,
|
||||
call,
|
||||
orgSlug: org.slug,
|
||||
workspaceSlug: workspace.slug,
|
||||
projectSlug: project.slug,
|
||||
orgId: org.orgId,
|
||||
workspaceId: workspace.workspaceId,
|
||||
projectId: project.projectId,
|
||||
slugs: { org: org.slug, workspace: workspace.slug, project: project.slug },
|
||||
}
|
||||
}
|
||||
|
||||
/** A signed-up user who joined the project's organization with a role. */
|
||||
async function addMember(project: SeededProject, email: string, role: "viewer" | "member" | "admin") {
|
||||
const actor = await signUp(email)
|
||||
const invite = (await project.call.callByName("tenancy.member.invite", {
|
||||
org: project.orgSlug,
|
||||
email,
|
||||
role,
|
||||
})) as { token: string }
|
||||
const call = caller(actor)
|
||||
await call.callByName("tenancy.invitation.accept", { org: project.orgSlug, token: invite.token })
|
||||
return { actor, call }
|
||||
}
|
||||
|
||||
return {
|
||||
registry,
|
||||
data,
|
||||
entities,
|
||||
store,
|
||||
queue,
|
||||
audit,
|
||||
usage,
|
||||
/** The OCR port the worker uses, observed, and the fake platform behind it. */
|
||||
ocr,
|
||||
platform,
|
||||
/** Every request the model was sent, as the gateway received it, and a way to change its answers. */
|
||||
model: { requests: modelRequests, respond: script },
|
||||
caller,
|
||||
signUp,
|
||||
deliver,
|
||||
runJobs,
|
||||
delivered,
|
||||
seedProject,
|
||||
addMember,
|
||||
}
|
||||
}
|
||||
|
||||
export type DocumentsApp = Awaited<ReturnType<typeof makeDocumentsApp>>
|
||||
|
||||
/** Some text as the bytes of a file, for an upload. */
|
||||
export const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text)
|
||||
|
||||
/** Uploads a file (a PDF unless told otherwise) as its project's owner and carries it through the relay and the worker. */
|
||||
export async function uploadAndProcess(
|
||||
app: DocumentsApp,
|
||||
project: SeededProject,
|
||||
filename = "report.pdf",
|
||||
contentType = "application/pdf",
|
||||
) {
|
||||
const file = await uploadViaProcedures(project.call, project.slugs, {
|
||||
filename,
|
||||
contentType,
|
||||
// The files module checks the first bytes against the type the upload declares, so a picture
|
||||
// starts with the PNG signature and a document with the PDF one.
|
||||
bytes:
|
||||
contentType === "image/png"
|
||||
? new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, ...bytesOf(`fixture ${filename}`)])
|
||||
: bytesOf(`%PDF-1.4 fixture ${filename}`),
|
||||
})
|
||||
await app.deliver()
|
||||
await app.runJobs()
|
||||
return file
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The test double of the AI gateway for the documents module: a deterministic reader, standing in for a
|
||||
// model behind `platform.ai.respond` of the fake platform.
|
||||
//
|
||||
// It answers the extraction request exactly as a well-behaved model would: it finds the page's text
|
||||
// between the tags the prompt names, takes every `label: value` line as a field, and quotes the line
|
||||
// itself. It never sees an image, never invents a word, and returns the same lines for the same text,
|
||||
// so a test (or the offline eval) that runs it measures the module's own code -- the request it builds,
|
||||
// the schema it parses, the verification, the cache, the proposals -- and not a model.
|
||||
|
||||
import { PAGE_TEXT_CLOSE, PAGE_TEXT_OPEN } from "@app/modules/documents"
|
||||
import type { FakeChatReply } from "@conusai-kit/testing/fake-platform"
|
||||
|
||||
const FIELD = /^\s*([^::]+?)\s*[::]\s*(\S.*?)\s*$/u
|
||||
const MAX_LABEL_CHARS = 60
|
||||
const MAX_LABEL_WORDS = 6
|
||||
const MAX_LINE_CHARS = 2_000
|
||||
|
||||
export interface ReadLineOfModel {
|
||||
label: string
|
||||
value: string
|
||||
quote: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The fields a reader of two line shapes finds in a page's text: `label: value`, and a table row
|
||||
* `first cell | other cells`, whose label is the first cell and whose value is the rest, as the prompt
|
||||
* asks a model to read it. The first row of a block of rows is the table's header, which names the
|
||||
* columns and is not a field.
|
||||
*/
|
||||
export function keyValueLines(text: string): ReadLineOfModel[] {
|
||||
const found: ReadLineOfModel[] = []
|
||||
let inTable = false
|
||||
for (const line of text.split("\n")) {
|
||||
if (line.includes(" | ")) {
|
||||
const header = !inTable
|
||||
inTable = true
|
||||
const [label, ...rest] = line.split(" | ").map((cell) => cell.trim())
|
||||
if (!header && label && rest.length > 0 && rest.every((cell) => cell !== "") && line.length <= MAX_LINE_CHARS) {
|
||||
found.push({ label, value: rest.join(" | "), quote: line.trim() })
|
||||
}
|
||||
continue
|
||||
}
|
||||
inTable = false
|
||||
const match = FIELD.exec(line)
|
||||
if (!match) continue
|
||||
const label = match[1] as string
|
||||
const value = match[2] as string
|
||||
if (label.length > MAX_LABEL_CHARS || label.split(/\s+/u).length > MAX_LABEL_WORDS) continue
|
||||
if (line.length > MAX_LINE_CHARS) continue
|
||||
found.push({ label, value, quote: line.trim() })
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
type Part = { type?: string; text?: string }
|
||||
|
||||
/** The page text of an extraction request: what lies between the prompt's tags in the user's message. */
|
||||
export function pageTextOf(request: Record<string, unknown>): string | undefined {
|
||||
const messages = (request.messages ?? []) as Array<{ role?: string; content?: string | Part[] }>
|
||||
const user = [...messages].reverse().find((message) => message.role === "user")
|
||||
const content = user?.content
|
||||
const text = typeof content === "string" ? content : (content ?? []).map((part) => part.text ?? "").join("\n")
|
||||
const from = text.indexOf(PAGE_TEXT_OPEN)
|
||||
const to = text.lastIndexOf(PAGE_TEXT_CLOSE)
|
||||
return from < 0 || to < from ? undefined : text.slice(from + PAGE_TEXT_OPEN.length + 1, to - 1)
|
||||
}
|
||||
|
||||
/** What an extraction request is answered with by the deterministic reader. */
|
||||
export const deterministicModel = (request: Record<string, unknown>): FakeChatReply => {
|
||||
const text = pageTextOf(request)
|
||||
return { content: JSON.stringify({ lines: text === undefined ? [] : keyValueLines(text) }) }
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// Real images for the logo tests: a PNG written by hand, so a test can break one chunk, and a WebP
|
||||
// that libwebp wrote. The kit's logo checks read structure and decode pixels, so a few magic bytes
|
||||
// no longer stand in for an image.
|
||||
|
||||
import { deflateSync, inflateSync } from "node:zlib"
|
||||
|
||||
const TABLE = (() => {
|
||||
const table = new Uint32Array(256)
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1
|
||||
table[n] = c >>> 0
|
||||
}
|
||||
return table
|
||||
})()
|
||||
|
||||
const crc32 = (bytes: Uint8Array): number => {
|
||||
let c = 0xffffffff
|
||||
for (const byte of bytes) c = (TABLE[(c ^ byte) & 0xff] as number) ^ (c >>> 8)
|
||||
return (c ^ 0xffffffff) >>> 0
|
||||
}
|
||||
|
||||
const concat = (...parts: Uint8Array[]): Buffer => Buffer.concat(parts)
|
||||
|
||||
export const chunk = (type: string, data: Uint8Array): Buffer => {
|
||||
const out = Buffer.alloc(12 + data.length)
|
||||
out.writeUInt32BE(data.length, 0)
|
||||
out.write(type, 4, "latin1")
|
||||
out.set(data, 8)
|
||||
out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length)
|
||||
return out
|
||||
}
|
||||
|
||||
const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
||||
|
||||
/** A PNG over `pixel(x, y)` -> [r, g, b, a]. */
|
||||
export function pngOf(width: number, height: number, pixel: (x: number, y: number) => readonly number[]): Buffer {
|
||||
const header = Buffer.alloc(13)
|
||||
header.writeUInt32BE(width, 0)
|
||||
header.writeUInt32BE(height, 4)
|
||||
header.set([8, 6, 0, 0, 0], 8)
|
||||
const rows = Buffer.alloc(height * (1 + width * 4))
|
||||
for (let y = 0; y < height; y++)
|
||||
for (let x = 0; x < width; x++) rows.set(pixel(x, y), y * (1 + width * 4) + 1 + x * 4)
|
||||
return concat(SIGNATURE, chunk("IHDR", header), chunk("IDAT", deflateSync(rows)), chunk("IEND", new Uint8Array()))
|
||||
}
|
||||
|
||||
/** A teal disc on transparent that fills the height. */
|
||||
export const markPng = (width: number, height: number, colour: readonly number[] = [0x80, 0xcd, 0xc6, 255]): Buffer =>
|
||||
pngOf(width, height, (x, y) => {
|
||||
const dx = x - width / 2
|
||||
const dy = y - height / 2
|
||||
return dx * dx + dy * dy < (Math.min(width, height) / 2) ** 2 ? colour : [0, 0, 0, 0]
|
||||
})
|
||||
|
||||
/** Pseudo-random pixels, which PNG cannot compress: a file about as large as its pixels. */
|
||||
export function noisePng(width: number, height: number): Buffer {
|
||||
let seed = 12345
|
||||
const next = () => {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0
|
||||
return seed >>> 24
|
||||
}
|
||||
return pngOf(width, height, () => [next(), next(), next(), 255])
|
||||
}
|
||||
|
||||
/** A lossless WebP of a teal disc, 400 x 100, written by libwebp. */
|
||||
export const MARK_WEBP = Buffer.from(
|
||||
"UklGRvwAAABXRUJQVlA4TO8AAAAvj8EYEA8wzQM4xvMf8HBU27bT0DNEwpeCNJCGlC/hD9NvKuXe91InEf135EaSInlgeU8DlfmEzM4yACXbFuj4j/SHeNfSX9Nfnu/A/oT+QQToH6RHgfxN+hfPj8CEJzn1l/Qfjs821P+i/1V6Fqj/Rf+r47OefxgAlmFCTjxw6IfOOGVjV5zS2pWmBLu2Uzq7jlN6u85T8NphUv7KDdPK/6D3cdV7G3chH3dsH5OHjwnKxyToY6JVzNmKWd5FwlDkHkW2UuQ3RUZU5FBF1lXkaUFmV/QCiu5B0W8oOhRBT6PoghR9k6LTEvRmdrq5DAA=",
|
||||
"base64",
|
||||
)
|
||||
|
||||
export const dimensions = (png: Uint8Array): { width: number; height: number } => ({
|
||||
width: Buffer.from(png).readUInt32BE(16),
|
||||
height: Buffer.from(png).readUInt32BE(20),
|
||||
})
|
||||
|
||||
/** The [r, g, b, a] of one pixel of an 8-bit RGBA, non-interlaced PNG (what the renderers write), by unfiltering its rows. */
|
||||
export function pixelAt(png: Uint8Array, x: number, y: number): number[] {
|
||||
const bytes = Buffer.from(png)
|
||||
const { width, height } = dimensions(bytes)
|
||||
if (bytes[24] !== 8 || bytes[25] !== 6 || bytes[28] !== 0)
|
||||
throw new Error("pixelAt reads 8-bit RGBA PNGs without interlace")
|
||||
const data: Buffer[] = []
|
||||
for (let at = 8; at < bytes.length; ) {
|
||||
const length = bytes.readUInt32BE(at)
|
||||
if (bytes.toString("latin1", at + 4, at + 8) === "IDAT") data.push(bytes.subarray(at + 8, at + 8 + length))
|
||||
at += 12 + length
|
||||
}
|
||||
const raw = inflateSync(Buffer.concat(data))
|
||||
const stride = width * 4
|
||||
const rows: Buffer[] = []
|
||||
for (let row = 0; row < height; row++) {
|
||||
const filter = raw[row * (stride + 1)] as number
|
||||
const line = Buffer.from(raw.subarray(row * (stride + 1) + 1, (row + 1) * (stride + 1)))
|
||||
const above = rows[row - 1]
|
||||
for (let i = 0; i < stride; i++) {
|
||||
const left = i >= 4 ? (line[i - 4] as number) : 0
|
||||
const up = above ? (above[i] as number) : 0
|
||||
const corner = above && i >= 4 ? (above[i - 4] as number) : 0
|
||||
const predictor = [0, left, up, (left + up) >> 1, paeth(left, up, corner)][filter] as number
|
||||
line[i] = ((line[i] as number) + predictor) & 0xff
|
||||
}
|
||||
rows.push(line)
|
||||
}
|
||||
const line = rows[y] as Buffer
|
||||
return [line[x * 4], line[x * 4 + 1], line[x * 4 + 2], line[x * 4 + 3]] as number[]
|
||||
}
|
||||
|
||||
function paeth(a: number, b: number, c: number): number {
|
||||
const p = a + b - c
|
||||
const [pa, pb, pc] = [Math.abs(p - a), Math.abs(p - b), Math.abs(p - c)]
|
||||
return pa <= pb && pa <= pc ? a : pb <= pc ? b : c
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The whole app of this template on the in-memory data source, wired the way apps/web and apps/worker
|
||||
// wire it, for the notes module's tests: the registry of every module, one caller per actor, the
|
||||
// outbox relay's behaviour and the worker's job runner.
|
||||
//
|
||||
// `deliver()` stands in for the outbox relay: it drains the outbox once, runs each record's
|
||||
// subscribers in a child container with the record's organization and a system actor, and marks it
|
||||
// published -- exactly once per subscriber, never more forgiving than the real relay. `runJobs()`
|
||||
// stands in for the worker's job runner. What neither does is retry, so a test of a redelivery
|
||||
// runs a handler again itself.
|
||||
|
||||
import { type Actor, buildRegistry, Container, createCaller, type TransportCaller } from "@conusai-kit/core"
|
||||
import { systemActor } from "@conusai-kit/core/worker"
|
||||
import {
|
||||
createMemoryDataSource,
|
||||
DATA_SOURCE,
|
||||
entitiesOf,
|
||||
JOB_QUEUE,
|
||||
OBJECT_STORE,
|
||||
SYSTEM_DATA,
|
||||
} from "@conusai-kit/data"
|
||||
import { FILES_AUDIT, FILES_REGISTRY, FILES_SCOPE, FILES_USAGE } from "@conusai-kit/files"
|
||||
import { PROPOSALS_REGISTRY } from "@conusai-kit/proposals"
|
||||
import { createTenancyHooks, TENANCY, users } from "@conusai-kit/tenancy"
|
||||
import {
|
||||
fakeJobQueue,
|
||||
fakeObjectStore,
|
||||
recordingAudit,
|
||||
recordingUsage,
|
||||
uploadViaProcedures,
|
||||
} from "@conusai-kit/testing"
|
||||
import { appModules } from "../../modules/index.js"
|
||||
import { NOTES_FLAGS, type NotesFlags } from "../../modules/notes/index.js"
|
||||
|
||||
export interface SeededProject {
|
||||
readonly owner: Actor
|
||||
readonly call: TransportCaller
|
||||
readonly orgSlug: string
|
||||
readonly orgId: string
|
||||
readonly slugs: { org: string; workspace: string; project: string }
|
||||
}
|
||||
|
||||
/** Standard base64 of some text, for the upload procedure's body. */
|
||||
export const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text)
|
||||
|
||||
export function makeNotesApp(options: { flags?: NotesFlags } = {}) {
|
||||
const registry = buildRegistry([...appModules])
|
||||
const entities = entitiesOf(registry, "postgres")
|
||||
const data = createMemoryDataSource({ entities })
|
||||
const store = fakeObjectStore()
|
||||
const queue = fakeJobQueue()
|
||||
const audit = recordingAudit()
|
||||
const usage = recordingUsage()
|
||||
|
||||
const root = new Container()
|
||||
root.value(DATA_SOURCE, data)
|
||||
root.value(SYSTEM_DATA, data)
|
||||
root.value(OBJECT_STORE, store)
|
||||
root.value(JOB_QUEUE, queue)
|
||||
root.value(FILES_REGISTRY, registry)
|
||||
root.value(FILES_AUDIT, audit)
|
||||
root.value(FILES_USAGE, usage)
|
||||
root.value(PROPOSALS_REGISTRY, registry)
|
||||
if (options.flags) root.value(NOTES_FLAGS, options.flags)
|
||||
|
||||
const caller = (actor: Actor): TransportCaller => {
|
||||
const tenancy = createTenancyHooks({ data, registry })
|
||||
const container = root.child()
|
||||
container.value(TENANCY, tenancy)
|
||||
return createCaller({ registry, container, tenancy }, { actor })
|
||||
}
|
||||
|
||||
const signUp = async (email: string): Promise<Actor> => {
|
||||
const user = await data.global(users).insert({
|
||||
issuer: "https://id.example.test",
|
||||
subject: email,
|
||||
email,
|
||||
displayName: email.split("@")[0] ?? email,
|
||||
locale: "en",
|
||||
})
|
||||
return { userId: user.id }
|
||||
}
|
||||
|
||||
/** Runs every subscriber of every record in the outbox, once each; answers how many ran. */
|
||||
const deliver = async (): Promise<number> => {
|
||||
let ran = 0
|
||||
for (const record of await data.outbox.pending(500)) {
|
||||
const entry = registry.events[record.name]
|
||||
for (const subscriber of entry?.subscribers ?? []) {
|
||||
const container = root.child()
|
||||
container.value(FILES_SCOPE, { level: "org", actor: systemActor("app.relay"), orgId: record.orgId })
|
||||
await subscriber.handler(
|
||||
{ name: record.name, payload: record.payload },
|
||||
{ container, correlationId: record.id },
|
||||
)
|
||||
ran += 1
|
||||
}
|
||||
await data.outbox.markPublished([record.id])
|
||||
}
|
||||
return ran
|
||||
}
|
||||
|
||||
/** The jobs not yet run, in the order they were enqueued. */
|
||||
let consumed = 0
|
||||
const runJobs = async (): Promise<number> => {
|
||||
let ran = 0
|
||||
while (consumed < queue.jobs.length) {
|
||||
const job = queue.jobs[consumed] as (typeof queue.jobs)[number]
|
||||
consumed += 1
|
||||
const registered = registry.jobs[job.job]
|
||||
if (!registered) throw new Error(`no module declares the job ${job.job}`)
|
||||
const container = root.child()
|
||||
container.value(TENANCY, createTenancyHooks({ data, registry }))
|
||||
container.value(FILES_SCOPE, { ...job.scope, actor: systemActor(job.job) })
|
||||
await registered.def.handler(registered.def.payload.parse(job.payload), {
|
||||
container,
|
||||
correlationId: job.id,
|
||||
attempt: 1,
|
||||
})
|
||||
ran += 1
|
||||
}
|
||||
return ran
|
||||
}
|
||||
|
||||
async function seedProject(slug: string, email: string): Promise<SeededProject> {
|
||||
const owner = await signUp(email)
|
||||
const call = caller(owner)
|
||||
const org = (await call.callByName("tenancy.org.create", { slug, name: slug })) as { orgId: string; slug: string }
|
||||
const workspace = (await call.callByName("tenancy.workspace.create", {
|
||||
org: slug,
|
||||
slug: `ws-${slug}`,
|
||||
name: `Workspace ${slug}`,
|
||||
visibility: "org",
|
||||
})) as { slug: string }
|
||||
const project = (await call.callByName("tenancy.project.create", {
|
||||
org: slug,
|
||||
workspace: workspace.slug,
|
||||
slug: `p-${slug}`,
|
||||
name: `Project ${slug}`,
|
||||
moduleKey: "notes",
|
||||
})) as { slug: string }
|
||||
return {
|
||||
owner,
|
||||
call,
|
||||
orgSlug: org.slug,
|
||||
orgId: org.orgId,
|
||||
slugs: { org: org.slug, workspace: workspace.slug, project: project.slug },
|
||||
}
|
||||
}
|
||||
|
||||
/** A signed-up user who joined the project's organization with a role. */
|
||||
async function addMember(project: SeededProject, email: string, role: "viewer" | "member" | "admin") {
|
||||
const actor = await signUp(email)
|
||||
const invite = (await project.call.callByName("tenancy.member.invite", {
|
||||
org: project.orgSlug,
|
||||
email,
|
||||
role,
|
||||
})) as { token: string }
|
||||
const call = caller(actor)
|
||||
await call.callByName("tenancy.invitation.accept", { org: project.orgSlug, token: invite.token })
|
||||
return { actor, call }
|
||||
}
|
||||
|
||||
/** Uploads a small text file through the files module, the way the page's form does. */
|
||||
async function upload(project: SeededProject, filename = "minutes.txt", text = "agreed") {
|
||||
return await uploadViaProcedures(project.call, project.slugs, {
|
||||
filename: filename,
|
||||
contentType: "text/plain",
|
||||
bytes: bytesOf(text),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
registry,
|
||||
/** The process container the procedures resolve their ports from (a seed script needs it). */
|
||||
root,
|
||||
data,
|
||||
entities,
|
||||
store,
|
||||
queue,
|
||||
audit,
|
||||
usage,
|
||||
caller,
|
||||
signUp,
|
||||
deliver,
|
||||
runJobs,
|
||||
seedProject,
|
||||
addMember,
|
||||
upload,
|
||||
}
|
||||
}
|
||||
|
||||
export type NotesApp = ReturnType<typeof makeNotesApp>
|
||||
@@ -0,0 +1,61 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The OCR path of a test: the real `@conusai-sdk/ocr` client (through kit-platform's `DocumentReader`)
|
||||
// over HTTP to the fake platform of @conusai-kit/testing, which accepts a job and answers it as the
|
||||
// platform's job API does. Nothing here stands in for the OCR path; `observed` only watches it, and can
|
||||
// make one call fail the way a network does, which the fake platform has no way to do.
|
||||
|
||||
import { type DocumentOcr, sdkDocumentOcr } from "@app/modules/documents"
|
||||
import { createDocumentReader } from "@conusai-kit/platform"
|
||||
import { startFakePlatform } from "@conusai-kit/testing/fake-platform"
|
||||
|
||||
export type FakePlatform = Awaited<ReturnType<typeof startFakePlatform>>
|
||||
|
||||
export interface ObservedOcr extends DocumentOcr {
|
||||
/** Every document accepted, with the idempotency key it came with. */
|
||||
readonly submitted: Array<{ jobId: number; filename: string; idempotencyKey: string }>
|
||||
/** Every `wait`, by job id: a retried job must resume its job and not submit another. */
|
||||
readonly waited: number[]
|
||||
/** The next `wait` throws this, once, before asking the platform anything. */
|
||||
failWaitOnce(error: Error): void
|
||||
}
|
||||
|
||||
export function observed(inner: DocumentOcr): ObservedOcr {
|
||||
const submitted: ObservedOcr["submitted"] = []
|
||||
const waited: number[] = []
|
||||
let nextWait: Error | undefined
|
||||
return {
|
||||
submitted,
|
||||
waited,
|
||||
failWaitOnce(error) {
|
||||
nextWait = error
|
||||
},
|
||||
async submit(bytes, options) {
|
||||
const accepted = await inner.submit(bytes, options)
|
||||
// The same key is the same job on the platform; a repeat is not a second submission.
|
||||
if (!submitted.some((entry) => entry.jobId === accepted.jobId)) {
|
||||
submitted.push({ jobId: accepted.jobId, filename: options.filename, idempotencyKey: options.idempotencyKey })
|
||||
}
|
||||
return accepted
|
||||
},
|
||||
page: (jobId, page, options) => inner.page(jobId, page, options),
|
||||
async wait(jobId, options) {
|
||||
waited.push(jobId)
|
||||
if (nextWait) {
|
||||
const error = nextWait
|
||||
nextWait = undefined
|
||||
throw error
|
||||
}
|
||||
return inner.wait(jobId, options)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** A fake platform that reads every document as `document`, and the app-side OCR port over it. */
|
||||
export async function startOcr(
|
||||
document: Record<string, unknown>,
|
||||
): Promise<{ platform: FakePlatform; ocr: ObservedOcr }> {
|
||||
const platform = await startFakePlatform()
|
||||
platform.ocr.setDocument(document)
|
||||
const reader = createDocumentReader({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "deployment-token" })
|
||||
return { platform, ocr: observed(sdkDocumentOcr(reader, { intervalMs: 1 })) }
|
||||
}
|
||||
Reference in New Issue
Block a user