ConusAI vendor v0.1.0-dev.20261027+9b1131fb
ci / gate (push) Waiting to run

This commit is contained in:
ConusAI vendor v0.1.0-dev.20261027+9b1131fb
2026-10-27 18:05:17 +00:00
commit 3102273ec7
538 changed files with 80203 additions and 0 deletions
+45
View File
@@ -0,0 +1,45 @@
{
"name": "@app/worker",
"version": "0.1.0",
"private": true,
"description": "The background process of this app: module jobs, module cron, the outbox relay, and the migrate job that owns the schema.",
"type": "module",
"scripts": {
"build": "tsc -p tsconfig.build.json",
"typecheck": "tsc -p tsconfig.json --noEmit",
"start": "node dist/main.js",
"seed": "node dist/seed-main.js",
"migrate": "node dist/migrate.js",
"clean": "rm -rf dist"
},
"dependencies": {
"@app/config": "workspace:*",
"@app/modules": "workspace:*",
"@conusai-kit/auth": "catalog:kit",
"@conusai-kit/auth-convex-better-auth": "catalog:kit",
"@conusai-kit/core": "catalog:kit",
"@conusai-kit/data": "catalog:kit",
"@conusai-kit/data-kysely": "catalog:kit",
"@conusai-kit/files": "catalog:kit",
"@conusai-kit/jobs-pgboss": "catalog:kit",
"@conusai-kit/platform": "catalog:kit",
"@conusai-kit/proposals": "catalog:kit",
"@conusai-kit/tenancy": "catalog:kit",
"@conusai-sdk/node-sdk": "catalog:",
"@conusai-sdk/observe-node": "catalog:",
"kysely": "catalog:",
"pg": "catalog:",
"pg-boss": "catalog:",
"zod": "catalog:",
"@conusai-kit/vector-qdrant": "catalog:kit",
"@qdrant/js-client-rest": "catalog:",
"@conusai-kit/agent": "catalog:kit",
"@conusai-kit/guides": "catalog:kit",
"@conusai-kit/support": "catalog:kit"
},
"devDependencies": {
"@types/node": "26.6.5",
"@types/pg": "catalog:",
"typescript": "5.9.3"
}
}
+18
View File
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The worker's part of the Convex Better Auth adapter (AUTH_PROVIDER=convex-better-auth): an account
// deletion records the provider whose account it ends, and the sweep that resumes one ends the Better
// Auth account at that step, as the web server would (apps/web/src/server/better-auth.ts).
// `kit template:prune --auth oidc` removes this file with the lines that call it.
import { ACCOUNT_IDENTITY_PROVIDER } from "@app/modules/account"
import { betterAuthAccountEndFromEnv } from "@conusai-kit/auth-convex-better-auth/account"
import type { Container } from "@conusai-kit/core"
/**
* Binds the account's end when Better Auth is the app's provider. Read at start-up, so a worker that
* lacks the deployment's variables stops by name instead of holding every deletion at that step.
*/
export function bindBetterAuthAccountEnd(container: Container, env: Record<string, string | undefined>): void {
if (env.AUTH_PROVIDER?.trim() !== "convex-better-auth") return
container.value(ACCOUNT_IDENTITY_PROVIDER, betterAuthAccountEndFromEnv(env))
}
+25
View File
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// kit:guides -- the guides module's build step; `kit template:prune --without guides` removes this file.
//
// Writes the app's code guides and its anchor catalogue into the image as one declarative, public
// file (ADR-045): `node apps/worker/dist/guides-bundle.js <out> [anchors]`, run by the Dockerfile
// after `kit guides:anchors`. conusai.json names <out> under `guides.bundle`, and after a healthy
// deploy the platform reads it out of the image and publishes it, with a credential no container
// ever holds. An invalid code guide or a broken catalogue fails the build.
import { existsSync } from "node:fs"
import { resolve } from "node:path"
import { appModules } from "@app/modules"
import { buildRegistry } from "@conusai-kit/core"
import { writeGuidesBundle } from "@conusai-kit/guides/deploy"
const [out, anchors] = process.argv.slice(2)
if (!out) {
console.error("guides-bundle: usage: node apps/worker/dist/guides-bundle.js <absolute out path> [anchors file]")
process.exit(64)
}
const anchorsFile = anchors && existsSync(anchors) ? resolve(anchors) : undefined
const bundle = writeGuidesBundle(buildRegistry([...appModules]), anchorsFile, out)
console.info(
`guides-bundle: wrote ${bundle.guides.length} code guide(s) and ${bundle.anchors.length} anchor(s) to ${out}`,
)
+24
View File
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The worker's healthcheck (compose.yaml). It reads the heartbeat the process writes rather than
// opening a database connection: a healthcheck that connects to Postgres every fifteen seconds
// reports the database's health, not the worker's, and holds a connection the app could use.
import { readFile } from "node:fs/promises"
const STALE_AFTER_MS = 45_000
async function main(): Promise<void> {
const file = process.env.WORKER_HEARTBEAT_FILE ?? "/tmp/worker-heartbeat"
const written = Number.parseInt(await readFile(file, "utf8"), 10)
if (!Number.isFinite(written)) throw new Error(`${file} holds no timestamp`)
const age = Date.now() - written
if (age > STALE_AFTER_MS) throw new Error(`the last heartbeat is ${Math.round(age / 1000)}s old`)
}
main().then(
() => process.exit(0),
(error: unknown) => {
console.error(`worker healthcheck: ${error instanceof Error ? error.message : String(error)}`)
process.exit(1)
},
)
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// What the worker tells observability about its work (spec §9.5): one span per job, and one log
// line per worker event, with a failed job reported as an error.
//
// Neither touches a module. The jobs a module declares in its manifest are wrapped before the
// registry is built, so a module's handler runs as the same function inside a span; the worker's own
// log callback turns the kit's events into lines. No payload is ever logged -- a job's payload
// belongs to a tenant (kit-jobs-pgboss) -- and nothing here carries a person's data.
import type { ServerObservability } from "@app/config"
import type { JobContext, ModuleManifest } from "@conusai-kit/core"
import { JOB_SCOPE } from "@conusai-kit/jobs-pgboss"
/** The tenant ids of the job's scope as span attributes: opaque identifiers, no names, no addresses. */
function scopeAttributes(context: JobContext): Record<string, string> {
try {
const scope = context.container.resolve(JOB_SCOPE)
return {
"conusai.org_id": scope.orgId,
...("workspaceId" in scope && typeof scope.workspaceId === "string"
? { "conusai.workspace_id": scope.workspaceId }
: {}),
...("projectId" in scope && typeof scope.projectId === "string" ? { "conusai.project_id": scope.projectId } : {}),
}
} catch {
// A container without a job scope (a test's): the span is still worth having without it.
return {}
}
}
/** The manifests with each module job's handler run inside a span named `job <name>`. */
export function traceJobs(modules: readonly ModuleManifest[], observe: ServerObservability): ModuleManifest[] {
return modules.map((module) => {
if (!module.jobs) return module
const jobs = Object.fromEntries(
Object.entries(module.jobs).map(([name, def]) => [
name,
{
...def,
handler: (payload: unknown, context: JobContext) =>
observe.span(
`job ${name}`,
{
"conusai.job": name,
"conusai.attempt": context.attempt,
"conusai.correlation_id": context.correlationId,
...scopeAttributes(context),
},
() => def.handler(payload, context),
),
},
]),
)
return { ...module, jobs }
})
}
/** The kit's worker events that carry an `error`; the rest are plain facts. */
interface WorkerEventLike {
readonly kind: string
readonly [field: string]: unknown
}
const FAILURES = new Set(["job.failed", "relay.failed", "relay.invalid-payload", "job.rejected", "relay.stuck"])
/** One line per event, and an error report for a failure. Only ids, names and counts are logged. */
export function logWorkerEvent(observe: ServerObservability, event: WorkerEventLike): void {
const { kind, error, ...fields } = event
const attributes: Record<string, string | number | boolean> = { event: kind }
for (const [name, value] of Object.entries(fields)) {
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") attributes[name] = value
}
if (!FAILURES.has(kind)) {
// The relay's pass runs every second or so, delivered or not: a line per pass is noise at `info`.
const line = `worker: ${kind}`
if (kind === "relay.pass") observe.log.debug(line, attributes)
else observe.log.info(line, attributes)
return
}
observe.log.error(`worker: ${kind}`, {
...attributes,
...(error instanceof Error ? { error: error.name } : {}),
})
if (error !== undefined) observe.captureException(error, { event: kind })
}
+80
View File
@@ -0,0 +1,80 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The worker process: module jobs, module cron and the outbox relay (spec §8).
//
// Everything here runs as a system actor, which is why this is the only app that may import
// @conusai-kit/core/worker (the boundary gate enforces it). A handler that must obey a rule such
// as "a suspended organization is read-only" calls the module's procedure through the kernel's
// caller rather than writing through a repository, so that rule lives in one place (AT-D19).
import { writeFile } from "node:fs/promises"
import { assertBoundByRowLevelSecurity, loadAppEnv, roleFacts, roleLine, startObservability } from "@app/config"
import { createWorker } from "@conusai-kit/jobs-pgboss"
import { logWorkerEvent } from "./instrument.js"
import { buildWorkerRuntime } from "./runtime.js"
import { appSchedules } from "./schedules.js"
/** The file the healthcheck reads. The relay touches it on every pass, so a stuck relay is visible. */
const heartbeatFile = (): string => process.env.WORKER_HEARTBEAT_FILE ?? "/tmp/worker-heartbeat"
async function main(): Promise<void> {
// Before anything else is built: a missing platform variable or an owner's URL stops the worker
// here, with a typed error that names it, and nothing has connected to anything yet.
const env = loadAppEnv("worker")
const observe = startObservability(env)
const runtime = buildWorkerRuntime(env, observe)
const { pools } = runtime
// Which role this process actually holds, before it serves a queue (decision AT-D20). A worker
// on the owner's login would write past every policy, so it refuses to start instead.
const runtimeRole = assertBoundByRowLevelSecurity(await roleFacts(pools.runtime), "POSTGRES_URL")
const readerRole = assertBoundByRowLevelSecurity(await roleFacts(pools.reader), "POSTGRES_READER_URL")
observe.log.info(roleLine("worker", runtimeRole))
observe.log.info(roleLine("worker reader", readerRole))
const worker = createWorker({
registry: runtime.registry,
boss: runtime.boss,
dataSource: runtime.data,
container: runtime.container,
schedules: appSchedules(),
log: (event) => {
// One line per event, no payload: a job's payload belongs to a tenant (kit-jobs-pgboss).
logWorkerEvent(observe, event)
void writeFile(heartbeatFile(), String(Date.now())).catch(() => undefined)
},
})
await worker.start()
await writeFile(heartbeatFile(), String(Date.now()))
observe.log.info(`worker: serving ${worker.queues.length} queue(s)`)
// The relay's own heartbeat, independent of whether anything was delivered: a worker with no
// work to do is healthy, and must say so.
const beat = setInterval(() => {
void writeFile(heartbeatFile(), String(Date.now())).catch(() => undefined)
}, 5_000)
beat.unref()
let stopping = false
const stop = (signal: string): void => {
if (stopping) return
stopping = true
observe.log.info(`worker: ${signal}, finishing the jobs in flight`)
void (async () => {
clearInterval(beat)
await worker.stop()
await runtime.close()
await observe.shutdown()
process.exit(0)
})()
}
process.on("SIGTERM", () => stop("SIGTERM"))
process.on("SIGINT", () => stop("SIGINT"))
}
main().catch((error: unknown) => {
// Nothing may have started when this runs (a bad environment), so it is written plainly: the
// message names the variable, and a typed error carries no secret by construction.
console.error(`worker: ${error instanceof Error ? error.message : String(error)}`)
process.exit(1)
})
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The migrate job (conusai.json: `"migrate": { "kind": "job", "postgres": { "migrate": true } }`).
//
// It is the only process that holds POSTGRES_MIGRATE_URL, the owner's URL, and the only one that
// writes the schema (decision AT-D20). Three things happen here, in this order:
//
// 1. the app's own forward-only migrations, newest last, from migrations/postgres
// 2. every module's declared migration folder, in the registry's dependency order
// 3. the job adapter's schema, and then the grants that let the runtime role use it and the
// global reader read the few tables it must read across tenants
// kit:guides:begin
// 4. with the guides module, the DEPRECATED compose hand-off of the code guides and the anchor
// catalogue. This app declares guides.bundle in conusai.json, so the platform publishes the
// bundle the image build wrote (apps/worker/src/guides-bundle.ts, ADR-045) and hands this job
// no credential: the step logs that it skipped. It stays only until the compose hand-off is
// removed, for an app that still has no guides.bundle.
// kit:guides:end
// kit:convex-better-auth:begin
// 5. with AUTH_PROVIDER=convex-better-auth, the Better Auth functions (better-auth/), deployed to
// the app's Better Auth Convex deployment with its SITE_URL and secret (better-auth/deploy.mjs)
// kit:convex-better-auth:end
//
// It exits non-zero when anything fails, which is what web and worker wait on:
// `depends_on: { migrate: { condition: service_completed_successfully } }` in compose.yaml
// (decision AT-D14 -- `kind: job` alone does not order a deployment).
import { spawnSync } from "node:child_process" // kit:convex-better-auth
import { existsSync } from "node:fs" // kit:guides
import { fileURLToPath } from "node:url" // kit:guides
import { assertOwnsSchema, loadAppEnv, roleFacts, roleLine } from "@app/config"
import { appModules } from "@app/modules"
import { buildRegistry } from "@conusai-kit/core"
import { runMigrations } from "@conusai-kit/data-kysely"
import { describeOutcome, publishGuidesAtDeploy } from "@conusai-kit/guides/deploy" // kit:guides
import { createPgBoss } from "@conusai-kit/jobs-pgboss"
import { Kysely, PostgresDialect } from "kysely"
import { Pool } from "pg"
import { jobSchemaGrants, moduleMigrations, readerGrants, sqlMigrations, vectorStoreMigration } from "./migrations.js"
async function main(): Promise<void> {
const env = loadAppEnv("migrate")
const postgres = env.postgres
if (!postgres) throw new Error("the selected profile has no Postgres migrations")
const owner = postgres.owner
if (!owner) throw new Error("the migrate job has no owner URL, which loadAppEnv refuses before this line")
const registry = buildRegistry([...appModules])
const pool = new Pool({ connectionString: owner, max: 2 })
const db = new Kysely<Record<string, never>>({ dialect: new PostgresDialect({ pool }) })
try {
// The owner, and this is the only process that may be it (decision AT-D20). It refuses to
// migrate as anything else: tables created by another role would be owned by it.
console.info(roleLine("migrate", assertOwnsSchema(await roleFacts(pool), "POSTGRES_MIGRATE_URL")))
const app = await sqlMigrations()
const modules = await moduleMigrations(registry)
const vectors = vectorStoreMigration(process.env)
const first = await runMigrations(db, [...app, ...modules, ...(vectors === null ? [] : [vectors])])
console.info(`migrate: applied ${first.applied.join(", ") || "nothing"}; skipped ${first.skipped.length}`)
// pg-boss owns its schema and creates it with the role in its connection string, which is
// this one. The runtime role is then granted what a transactional enqueue needs, in the same
// step that created the schema -- a grant left for later is a grant nobody runs.
const boss = createPgBoss({ connectionString: owner })
await boss.start()
await boss.stop({ graceful: true })
const grants = await runMigrations(db, [readerGrants(postgres.reader), jobSchemaGrants(postgres.runtime)])
console.info(`migrate: applied ${grants.applied.join(", ") || "nothing"}`)
// kit:guides:begin -- after the schema, so a refused publication never leaves a half-migrated
// database; a refusal is logged and the deployment goes on (the console shows the guide's state).
const anchors = new URL("../../../guides-anchors.json", import.meta.url)
const guides = await publishGuidesAtDeploy({
registry,
env: {
CONUSAI_API_URL: process.env.CONUSAI_API_URL,
CONUSAI_GUIDES_PUBLISH_TOKEN: process.env.CONUSAI_GUIDES_PUBLISH_TOKEN,
CONUSAI_GUIDES_DEPLOYMENT: process.env.CONUSAI_GUIDES_DEPLOYMENT,
},
...(existsSync(anchors) ? { anchorsFile: fileURLToPath(anchors) } : {}),
})
console.info(`migrate: ${describeOutcome(guides)}`)
// kit:guides:end
// kit:convex-better-auth:begin -- a failed deploy fails the job, so web and worker never start
// against a Better Auth deployment that cannot answer their session checks.
if (process.env.AUTH_PROVIDER?.trim() === "convex-better-auth") {
const script = new URL("../../../better-auth/deploy.mjs", import.meta.url).pathname
const deployed = spawnSync(process.execPath, [script], { env: process.env, stdio: "inherit" })
if (deployed.error || deployed.status !== 0) throw new Error("the Better Auth functions were not deployed")
}
// kit:convex-better-auth:end
} finally {
await db.destroy()
}
}
main().then(
() => process.exit(0),
(error: unknown) => {
console.error(`migrate: ${error instanceof Error ? error.message : String(error)}`)
process.exit(1)
},
)
+139
View File
@@ -0,0 +1,139 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The app's forward-only migrations for the Postgres profile (spec §13).
//
// 0001_kernel.sql is committed: it is the schema `schemaStatements()` generated for the kernel's
// tables once, and it stays byte-identical afterwards, because the migration runner refuses a
// migration whose text changed after it ran. A module added later ships its own numbered file;
// `test/migrations.test.ts` fails when an entity of the registry has no table in any of them.
//
// The reader grants cannot be a committed file: they name the global reader's role, which is
// derived from the database and known only from POSTGRES_READER_URL at run time (data-services
// DS21). They are therefore built here, from that one name.
import { readFile } from "node:fs/promises"
import {
appMigrationFiles,
GRANT_MIGRATIONS,
type MigrationFile,
moduleMigrationFiles,
roleOf,
vectorDimensionsOf,
vectorStoreMigrationName,
} from "@app/config"
import { AUTH_CROSS_TENANT_READS } from "@conusai-kit/auth"
import type { Registry } from "@conusai-kit/core"
import { type Migration, readerMigration, schemaStatements, vectorMigration } from "@conusai-kit/data-kysely"
import { TENANCY_CROSS_TENANT_READS } from "@conusai-kit/tenancy"
/** A migration file as the runner takes it: its recorded name and its statements. */
async function read(file: MigrationFile): Promise<Migration> {
return { name: file.name, statements: splitStatements(await readFile(file.path, "utf8")) }
}
/**
* Every committed migration, in file-name order, which is the order they were written in. Which
* files there are is config's (`appMigrationFiles`), because the web shell's readiness probe asks
* the same question of the same folder.
*/
export async function sqlMigrations(): Promise<Migration[]> {
return Promise.all((await appMigrationFiles()).map(read))
}
/**
* Splits a migration file into statements on `;` at the end of a line, which is the only
* separator these files use. A `$$`-quoted body is kept whole.
*/
export function splitStatements(sql: string): string[] {
const withoutComments = sql
.split("\n")
.filter((line) => !line.trimStart().startsWith("--"))
.join("\n")
const statements: string[] = []
let current = ""
let inDollar = false
for (const part of withoutComments.split(/(\$\$)/)) {
if (part === "$$") {
inDollar = !inDollar
current += part
continue
}
if (inDollar) {
current += part
continue
}
const pieces = part.split(";")
for (const [index, piece] of pieces.entries()) {
current += piece
if (index < pieces.length - 1) {
if (current.trim()) statements.push(current.trim())
current = ""
}
}
}
if (current.trim()) statements.push(current.trim())
return statements
}
/**
* Exactly what the global reader may read (decision AT-D20), and nothing else. It is not a list
* kept here: each kit module declares the cross-tenant reads it makes (`crossTenantRead`, kit-data)
* beside the code that makes them, a test in the module fails when the two disagree, and this is the
* union of those declarations:
*
* org_memberships which organizations a user is a member of, the one question that belongs
* to no organization (kit-tenancy, read by tenancy.me.get)
* projects the projects the app serves, which the worker lists to fan a scheduled job
* (`files.recover`) out to each of them (kit-tenancy `servedProjects`)
* auth_api_keys the lookup of an API key by its hash, before any scope exists (kit-auth)
*
* and, added by `readerMigration` itself, the outbox the relay drains across organizations. Its
* `update` is granted on the relay's bookkeeping columns, not on the table: a grant on the whole row
* would let it rewrite an event's organization, name or payload -- which the relay then hands to a
* subscriber running as a system actor.
*
* Each of those tables carries FORCE row-level security on `app.org_id`, and the reader owns
* nothing, so the policy binds it as it binds the runtime role: without a policy of its own the
* reader would see zero rows. The policies `readerMigration` writes are `to` the reader alone, so
* the runtime role keeps exactly the tenant wall it had.
*/
export const CROSS_TENANT_READS = [...TENANCY_CROSS_TENANT_READS, ...AUTH_CROSS_TENANT_READS]
export function readerGrants(readerUrl: string): Migration {
return readerMigration(roleOf(readerUrl), CROSS_TENANT_READS, { name: GRANT_MIGRATIONS[0] })
}
/** What the runtime role needs in pg-boss's schema, once the job adapter has created it. */
export function jobSchemaGrants(runtimeUrl: string, schema = "pgboss"): Migration {
const runtime = `"${roleOf(runtimeUrl)}"`
return {
name: GRANT_MIGRATIONS[1],
statements: [
`grant usage on schema "${schema}" to ${runtime}`,
`grant select, insert, update, delete on all tables in schema "${schema}" to ${runtime}`,
`grant usage, select on all sequences in schema "${schema}" to ${runtime}`,
`alter default privileges in schema "${schema}" grant select, insert, update, delete on tables to ${runtime}`,
`alter default privileges in schema "${schema}" grant usage, select on sequences to ${runtime}`,
],
}
}
/** Each module's SQL files, named `<module>/<file>` so two modules cannot collide. */
export async function moduleMigrations(registry: Registry): Promise<Migration[]> {
return Promise.all((await moduleMigrationFiles(registry)).map(read))
}
/**
* The kernel schema as the committed `0001_kernel.sql` holds it: one statement per paragraph, so
* a diff of the file reads as a diff of the schema (test/kernel-migration.test.ts).
*/
export function kernelSchema(entities: Parameters<typeof schemaStatements>[0]): string {
return `${schemaStatements(entities)
.map((statement) => `${statement};`)
.join("\n\n")}\n`
}
/** The runtime-built migration that creates the pgvector table, when this deployment needs one. */
export function vectorStoreMigration(environment: Record<string, string | undefined> = process.env): Migration | null {
const name = vectorStoreMigrationName(environment)
return name === null ? null : { name, statements: vectorMigration(vectorDimensionsOf(environment)).statements }
}
+221
View File
@@ -0,0 +1,221 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The ports the kit's modules and this app's modules resolve by token, bound for the worker.
//
// A module resolves a port by token and never imports an adapter (spec §5.6), so the adapters are
// chosen here, in the composition root, from the deployment contract. The platform's services
// themselves are built by `platformClients` (config/src/platform.ts), the same function the web
// shell calls, so the two processes cannot talk to different services. Three kinds of binding:
//
// - the platform's own services: Blob for the files module's object store, the audit log and the
// usage meter for the subscribers, the OCR service and the AI gateway for the documents job;
// - the registry, which the files and proposals modules read their extension points from;
// - the scope of a job or an outbox delivery, which the worker binds per run as `JOB_SCOPE`
// (kit-jobs-pgboss) and the modules that cannot import that adapter resolve through their own
// token (decision AT-F6). An alias is a factory that reads the per-run binding, so it is
// registered once, on the process's container.
import { excludedPageClassNamesOf, NO_VECTOR_STORE_REASON, type PlatformClients } from "@app/config"
import { ACCOUNT_REGISTRY } from "@app/modules/account"
import { AGENT_SCOPE, actionScopeOf, registryAgentTools } from "@app/modules/agent"
import {
DOCUMENT_EXCLUDED_PAGE_CLASSES,
DOCUMENT_MODEL,
DOCUMENT_OCR,
DOCUMENTS_SCOPE,
MODEL_GATEWAY,
pageClassesOf,
sdkDocumentOcr,
} from "@app/modules/documents"
import {
KNOWLEDGE_EMBEDDINGS,
KNOWLEDGE_SCOPE,
KNOWLEDGE_SEARCH,
platformKnowledgeSearch,
unavailableKnowledgeSearch,
vectorKnowledgeSearch,
} from "@app/modules/knowledge"
import { AGENT_TOOLS } from "@conusai-kit/agent"
import type { Container, Registry } from "@conusai-kit/core"
import { DATA_SOURCE, OBJECT_STORE, VECTOR_STORE } from "@conusai-kit/data"
import type { KyselyDataSource } from "@conusai-kit/data-kysely"
import { createKyselyVectorStore } from "@conusai-kit/data-kysely"
import {
FILES_AUDIT,
FILES_REGISTRY,
FILES_SCOPE,
FILES_USAGE,
type FilesAuditSink,
filesDeps,
platformObjectStore,
} from "@conusai-kit/files"
import { GUIDES_PLATFORM, GUIDES_REGISTRY, guidesPlatform } from "@conusai-kit/guides" // kit:guides
import { JOB_SCOPE } from "@conusai-kit/jobs-pgboss"
import { createKnowledge } from "@conusai-kit/platform"
import { PROPOSALS_AUDIT, PROPOSALS_REGISTRY, PROPOSALS_SCOPE } from "@conusai-kit/proposals"
import { SUPPORT_PLATFORM, supportPlatform } from "@conusai-kit/support" // kit:support
import { createQdrantVectorStore, ensureQdrantCollection } from "@conusai-kit/vector-qdrant"
import { QdrantClient } from "@qdrant/js-client-rest"
import { bindBetterAuthAccountEnd } from "./better-auth.js" // kit:convex-better-auth
export function bindModulePorts(container: Container, platform: PlatformClients, registry: Registry): void {
container.value(OBJECT_STORE, platformObjectStore({ blob: platform.objectStore }))
container.value(FILES_REGISTRY, registry)
container.value(PROPOSALS_REGISTRY, registry)
// The account module's sweep resumes a deletion a module's erasure held up: it reads the installed
// modules' shares from the registry and reaches their services through the same ports the web binds.
container.value(ACCOUNT_REGISTRY, registry)
// The sweep resumes a deletion at the step that ends the person's Better Auth account.
bindBetterAuthAccountEnd(container, process.env) // kit:convex-better-auth
// kit:guides:begin -- the guides module's erasure, for a resumed account deletion.
container.value(GUIDES_REGISTRY, registry)
container.value(
GUIDES_PLATFORM,
guidesPlatform({
apiUrl: platform.deployment.CONUSAI_API_URL,
token: platform.deployment.CONUSAI_API_TOKEN,
fake: platform.mode === "fake",
}),
)
// kit:guides:end
// kit:support:begin -- the support module's erasure, for a resumed account deletion.
container.value(
SUPPORT_PLATFORM,
supportPlatform({
apiUrl: platform.deployment.CONUSAI_API_URL,
token: platform.deployment.CONUSAI_API_TOKEN,
channel: process.env.CONUSAI_SUPPORT_CHANNEL,
fake: platform.mode === "fake",
}),
)
// kit:support:end
// The modules' sinks take the action as a plain string; the registry the audit client was built
// from (config's `auditEventsOf`) is what makes it a known one, and the client refuses the rest.
const sink: FilesAuditSink = {
log: (action, fields) => platform.audit.log(action, fields as Parameters<typeof platform.audit.log>[1]),
}
container.value(FILES_AUDIT, sink)
container.value(PROPOSALS_AUDIT, sink)
container.value(FILES_USAGE, {
track: (event) => platform.usage.track(event),
flush: async () => {
await platform.usage.flush()
},
})
// The platform's asynchronous OCR job API: a job submits and polls, a request never does.
container.value(DOCUMENT_OCR, sdkDocumentOcr(platform.documents))
// The AI gateway the documents job extracts lines through, and the model it asks for (AI_MODEL).
container.value(MODEL_GATEWAY, platform.models)
container.value(DOCUMENT_MODEL, platform.model)
// The page classes the extraction never asks the model about; unset, the module's default (drawings and empty pages).
const excluded = excludedPageClassNamesOf()
if (excluded) container.value(DOCUMENT_EXCLUDED_PAGE_CLASSES, pageClassesOf(excluded))
// The per-run scope, under each module's own name for it.
for (const alias of [FILES_SCOPE, PROPOSALS_SCOPE, DOCUMENTS_SCOPE, KNOWLEDGE_SCOPE, AGENT_SCOPE]) {
container.register(alias, (c) => c.resolve(JOB_SCOPE), { lifetime: "scoped" })
}
// The knowledge module's search backend and the agent runtime's tools, bound for the worker.
bindKnowledgePorts(container, platform)
bindAgentPorts(container, platform)
}
const KNOWLEDGE_COLLECTION = "knowledge"
/**
* The knowledge module's ports. The vector store binds the Qdrant adapter when a Qdrant service is
* linked (QDRANT_URL), the pgvector adapter over the app's own Postgres only when chosen
* (KNOWLEDGE_VECTOR_STORE=pgvector), and otherwise no store: the search service then answers
* unavailable with the reason. The search service binds the local vectors backend or the managed
* documents-index service per KNOWLEDGE_BACKEND.
* Indexing and search jobs run in this process only; a request writes the outbox event and answers.
*/
function bindKnowledgePorts(container: Container, platform: PlatformClients): void {
container.value(KNOWLEDGE_EMBEDDINGS, {
embed: (texts, options) => {
const embed = platform.models.embed
if (!embed) {
throw new Error("the AI gateway has no embeddings model configured (AI_EMBEDDING_MODEL)")
}
return embed.call(platform.models, texts, options)
},
})
if (platform.knowledge.backend === "platform") {
container.register(KNOWLEDGE_SEARCH, (c) =>
platformKnowledgeSearch(createKnowledge(platform.deployment), filesDeps(c)),
)
return
}
if (platform.knowledge.vectorStore === "qdrant" && platform.knowledge.qdrantUrl) {
const client = new QdrantClient({
url: platform.knowledge.qdrantUrl,
...(platform.knowledge.qdrantApiKey === undefined ? {} : { apiKey: platform.knowledge.qdrantApiKey }),
})
// The collection exists, with its payload indexes, before the worker serves a queue.
void ensureQdrantCollection(client, { collection: KNOWLEDGE_COLLECTION, dimensions: platform.knowledge.dimensions })
const store = createQdrantVectorStore({
client,
collection: KNOWLEDGE_COLLECTION,
dimensions: platform.knowledge.dimensions,
})
container.value(VECTOR_STORE, store)
container.value(KNOWLEDGE_SEARCH, vectorKnowledgeSearch({ embeddings: knowledgeEmbeddings(platform), store }))
return
}
if (platform.knowledge.vectorStore !== "pgvector") {
// No store chosen (no Qdrant linked, pgvector not opted into): the honest unavailable service,
// never a silent pgvector default on a Postgres that may not have the extension.
container.value(KNOWLEDGE_SEARCH, unavailableKnowledgeSearch(NO_VECTOR_STORE_REASON))
return
}
// Lazy, like the web shell's: the pgvector adapter resolves the process data source only when a
// job actually searches or indexes, which keeps a bare container bindable in tests.
container.register(VECTOR_STORE, (c) => {
const data = c.resolve(DATA_SOURCE) as KyselyDataSource
return createKyselyVectorStore({
db: data.db,
inScope: (scope, run) => data.inScope(scope, run),
dimensions: platform.knowledge.dimensions,
})
})
container.register(KNOWLEDGE_SEARCH, (c) =>
vectorKnowledgeSearch({ embeddings: knowledgeEmbeddings(platform), store: c.resolve(VECTOR_STORE) }),
)
}
const knowledgeEmbeddings = (platform: PlatformClients): import("@app/modules/knowledge").KnowledgeEmbeddings => ({
embed: (texts, options) => {
const embed = platform.models.embed
if (!embed) {
throw new Error("the AI gateway has no embeddings model configured (AI_EMBEDDING_MODEL)")
}
return embed.call(platform.models, texts, options)
},
})
/** The agent runtime's tools: the organization's registry, with the app's scoped credentials. */
function bindAgentPorts(container: Container, platform: PlatformClients): void {
container.value(
AGENT_TOOLS,
registryAgentTools({
apiUrl: platform.deployment.CONUSAI_API_URL,
token: platform.deployment.CONUSAI_API_TOKEN,
orgId: platform.deployment.CONUSAI_ORG_ID ?? "0",
// Resolved per call: jobs of different tenants share this one process binding, and the
// registry's tool-actions API is addressed by slugs, not ids.
scope: async () => {
const scope = container.resolve(AGENT_SCOPE)
if (scope.level !== "project") throw new Error("agent tools resolve in a project scope")
return actionScopeOf(container.resolve(DATA_SOURCE), scope)
},
}),
)
}
+106
View File
@@ -0,0 +1,106 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The worker's composition root (spec §2: apps/* may import everything and contain nothing else).
//
// It is the same wiring the web shell does, with two differences: the container is the process's
// own rather than one per request, and this is the only app that may import
// @conusai-kit/core/worker, because only work nobody started acts as a system actor.
import { type AppEnv, type PlatformClients, PROFILE, platformClients, type ServerObservability } from "@app/config"
import { appModules } from "@app/modules"
import { buildRegistry, Container, type Registry } from "@conusai-kit/core"
import { DATA_SOURCE, entitiesOf, IDEMPOTENCY, JOB_QUEUE, OUTBOX, SYSTEM_DATA } from "@conusai-kit/data"
import { createKyselyDataSource, type KyselyDataSource } from "@conusai-kit/data-kysely"
import {
createPgBoss,
createPgBossJobQueue,
fromKyselyTransaction,
KIT_JOBS_ENTITIES,
withJobTransaction,
} from "@conusai-kit/jobs-pgboss"
import { createTenancyHooks, TENANCY } from "@conusai-kit/tenancy"
import { Pool } from "pg"
import { traceJobs } from "./instrument.js"
import { bindModulePorts } from "./ports.js"
export interface WorkerRuntime {
readonly registry: Registry
readonly data: KyselyDataSource
/** The two pools, for the role check at start-up: everything else goes through the data source. */
readonly pools: { readonly runtime: Pool; readonly reader: Pool }
readonly container: Container
/** The platform clients bound into the container: the worker flushes the usage meter on the way out. */
readonly platform: PlatformClients
readonly boss: ReturnType<typeof createPgBoss>
close(): Promise<void>
}
/**
* The tables this app's schema is made of: the kernel's own, every module's entities for this
* profile, and the one table the job adapter owns. The migrate job builds the schema from exactly
* this list, so a module's table cannot be missing from it.
*/
export function appEntities(registry: Registry) {
return [...entitiesOf(registry, PROFILE), ...KIT_JOBS_ENTITIES]
}
/** Two pools, two roles (decision AT-D20): the runtime role for everything, the reader for the reads that have no scope. */
export function appPools(env: AppEnv): { runtime: Pool; reader: Pool } {
const postgres = env.postgres
if (!postgres) throw new Error("The selected profile has no Postgres worker")
return {
runtime: new Pool({ connectionString: postgres.runtime, max: 10 }),
reader: new Pool({ connectionString: postgres.reader, max: 4 }),
}
}
export function buildWorkerRuntime(env: AppEnv, observe: ServerObservability): WorkerRuntime {
// Each module job runs inside a span (instrument.ts); the handlers themselves are the modules own.
const registry = buildRegistry(traceJobs(appModules, observe))
const platform = platformClients(env, registry)
const pools = appPools(env)
const postgres = env.postgres
if (!postgres) throw new Error("The selected profile has no Postgres worker")
const data = createKyselyDataSource({
entities: appEntities(registry),
pool: pools.runtime,
readerPool: pools.reader,
// An enqueue made while a transaction is open joins it: the job exists exactly when the
// write that asked for it commits. The files module's processors enqueue from an outbox
// delivery and refuse to without this (kit-files: "requires the shared delivery transaction").
transactionContext: (trx, run) => withJobTransaction(fromKyselyTransaction(trx), run),
})
// pg-boss keeps its own schema and its own pool. The runtime role may use that schema, which is
// what a transactional enqueue needs; creating it is the migrate job's work.
const boss = createPgBoss({ connectionString: postgres.runtime })
const queue = createPgBossJobQueue({ boss })
const container = new Container()
container.value(DATA_SOURCE, data)
// The two ports that have no tenant scope are bound in this process alone: a request container
// does not get them, so module code cannot read across tenants from a request (kit-data).
container.value(SYSTEM_DATA, data)
container.value(OUTBOX, data.outbox)
container.value(IDEMPOTENCY, data.idempotency)
container.value(JOB_QUEUE, queue)
// A handler that calls a procedure needs the resolver; it is registered per job in the child
// container the worker builds, because the memo of roles must not outlive one run.
container.register(TENANCY, () => createTenancyHooks({ data, registry }), { lifetime: "transient" })
// The platform's services and the modules' scope tokens (ports.ts).
bindModulePorts(container, platform, registry)
return {
registry,
data,
pools,
platform,
container,
boss,
close: async () => {
await boss.stop({ graceful: true }).catch(() => undefined)
// Usage events batch in memory: send what is queued before the process goes.
await platform.usage.close().catch(() => undefined)
// Closes both pools, once (kit-data-kysely): ending them again here is an error.
await data.close()
},
}
}
+25
View File
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The module cron of this app: what the worker runs on a clock, not because anyone asked.
//
// A kit module declares its schedules beside its manifest (kit-core's manifest carries handlers and
// no timing); the composition root is what turns them on, so a module that is not in this list is
// not scheduled and the registry says so at boot (`installSchedules` refuses a schedule of a job no
// module declares). Without the first line nothing ever finishes an upload whose request died, and
// every crash boundary of the files module stays open however good its recovery is.
import { ACCOUNT_SCHEDULES } from "@app/modules/account"
import { FILES_SCHEDULES } from "@conusai-kit/files"
import { defineSchedules, type ModuleSchedule } from "@conusai-kit/jobs-pgboss"
export function appSchedules(): readonly ModuleSchedule[] {
return [
// `files.recover`, every five minutes, for every project of every active organization: the
// worker lists the projects through the global reader and enqueues one run for each.
...defineSchedules("files", FILES_SCHEDULES),
// `account.deletion.sweep`, every five minutes and in no organization: resumes an account deletion
// whose module erasure was unavailable (the deletion ledger is its whole state). And
// `account.sessions.revoke`, every minute: delivers the revocations of signed-out app sessions'
// derived service sessions that a module's platform has not confirmed (the revocation outbox).
...defineSchedules("account", ACCOUNT_SCHEDULES),
]
}
+42
View File
@@ -0,0 +1,42 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// `pnpm seed`: the demo data (seed.ts) written into the database a development run uses. It runs as
// the runtime role, like the web shell, through the same use cases, so the schema must be migrated
// first (`pnpm migrate`; `pnpm dev` does both). It refuses a production run: demo people with known
// addresses in a real deployment's database is a mistake no flag should make easy.
import { loadAppEnv, PlatformEnvError, quietObservability } from "@app/config"
import { buildWorkerRuntime } from "./runtime.js"
import { DEMO_PEOPLE, seedDemo } from "./seed.js"
async function main(): Promise<void> {
const env = loadAppEnv("worker")
if (env.production) {
throw new PlatformEnvError(
"NODE_ENV",
"the demo data is for development: it is refused in a production run (NODE_ENV is neither development nor test)",
)
}
const runtime = buildWorkerRuntime(env, quietObservability(env))
try {
const summary = await seedDemo({ data: runtime.data, registry: runtime.registry, container: runtime.container })
const base = env.deployment.CONUSAI_PUBLIC_URL ?? "http://127.0.0.1:3000"
console.info(
summary.created.length === 0
? "seed: the demo data was already there"
: `seed: created ${summary.created.join("; ")}`,
)
for (const person of DEMO_PEOPLE) {
console.info(`seed: ${person.name}: ${base}/auth/dev-login?email=${person.email}`)
}
} finally {
await runtime.close()
}
}
main().then(
() => process.exit(0),
(error: unknown) => {
console.error(`seed: ${error instanceof Error ? error.message : String(error)}`)
process.exit(1)
},
)
+168
View File
@@ -0,0 +1,168 @@
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The demo data of a development run: two organizations, one of them with a tenant brand, three
// people who belong to them in different roles, and a project with a few notes -- enough to see the
// organization switcher, the roles, a branded organization and a module's pages the first time the
// app starts.
//
// It is written with the kit and nothing else. People come into being the way they do at sign-in
// (`upsertUser`, as `@conusai-kit/auth` calls it); everything after that is a procedure called as
// the person who may call it, through the kernel's own caller, so the permission checks, the
// last-owner rule, the invitation single use and the tenancy walls all apply to the seed exactly as
// they apply to a request. There is no SQL here and no row written past a use case.
//
// Running it twice changes nothing: every step first asks whether it is already there.
import { DEV_LOGIN_ISSUER, upsertUser } from "@conusai-kit/auth"
import { type Actor, type Container, createCaller, type Registry, type TransportCaller } from "@conusai-kit/core"
import { DATA_SOURCE, type DataSource } from "@conusai-kit/data"
import { createTenancyHooks, TENANCY } from "@conusai-kit/tenancy"
/** The people of the demo. They sign in at `/auth/dev-login?email=<email>` in a development run. */
export const DEMO_PEOPLE = [
{ email: "ada@example.test", name: "Ada Lovelace" },
{ email: "grace@example.test", name: "Grace Hopper" },
{ email: "linus@example.test", name: "Linus Torvalds" },
] as const
export const DEMO_ORGANIZATIONS = {
/** The default look: the app's own brand. */
acme: { slug: "acme-review", name: "Acme Review" },
/** A tenant brand: its own name and colours on every page. */
northwind: {
slug: "northwind",
name: "Northwind Traders",
brand: { displayName: "Northwind", primary: "#1a1a8c", accent: "#f4c430" },
},
} as const
const WORKSPACE = "general"
const PROJECT = { slug: "demo", name: "Demo project" }
export interface SeedDeps {
readonly data: DataSource
readonly registry: Registry
/** The process's container: the ports the modules' procedures resolve (Blob, the queue, ...). */
readonly container: Container
}
export interface SeedSummary {
/** What this run created; empty when everything was already there. */
readonly created: string[]
/** The addresses to sign in at, by person. */
readonly people: readonly { email: string; name: string }[]
readonly organizations: readonly { slug: string; name: string }[]
}
type Role = "viewer" | "member" | "admin" | "owner"
export async function seedDemo(deps: SeedDeps): Promise<SeedSummary> {
const { data, registry } = deps
const created: string[] = []
const actors = new Map<string, Actor>()
for (const person of DEMO_PEOPLE) {
const user = await upsertUser(data, {
issuer: DEV_LOGIN_ISSUER,
subject: person.email,
email: person.email,
displayName: person.name,
locale: "en",
})
actors.set(person.email, { userId: user.id })
}
/** A caller for one person, with its own tenancy hooks: the memo of roles must not outlive a call. */
const as = (email: string): TransportCaller => {
const actor = actors.get(email)
if (!actor) throw new Error(`the demo has no person ${email}`)
const tenancy = createTenancyHooks({ data, registry })
const container = deps.container.child()
container.value(DATA_SOURCE, data)
container.value(TENANCY, tenancy)
return createCaller({ registry, container, tenancy }, { actor })
}
const rolesOf = async (email: string): Promise<Map<string, Role>> => {
const me = (await as(email).callByName("tenancy.me.get", undefined)) as {
organizations: { slug: string; role: Role }[]
}
return new Map(me.organizations.map((entry) => [entry.slug, entry.role]))
}
/** Creates an organization for its owner unless they already have it. */
const organization = async (owner: string, org: { slug: string; name: string }): Promise<void> => {
if ((await rolesOf(owner)).has(org.slug)) return
await as(owner).callByName("tenancy.org.create", { slug: org.slug, name: org.name })
created.push(`organization ${org.slug}`)
}
/** Brings a person into an organization the way anybody joins one: an invitation, accepted by them. */
const member = async (owner: string, org: string, email: string, role: Role): Promise<void> => {
if ((await rolesOf(email)).has(org)) return
const invitation = (await as(owner).callByName("tenancy.member.invite", { org, email, role })) as { token: string }
await as(email).callByName("tenancy.invitation.accept", { org, token: invitation.token })
created.push(`${email} as ${role} in ${org}`)
}
const project = async (owner: string, org: string, moduleKey: string, slug: string, name: string): Promise<void> => {
const listed = (await as(owner).callByName("tenancy.project.list", { org, workspace: WORKSPACE })) as {
projects: { slug: string }[]
}
if (listed.projects.some((entry) => entry.slug === slug)) return
await as(owner).callByName("tenancy.project.create", { org, workspace: WORKSPACE, slug, name, moduleKey })
created.push(`project ${org}/${WORKSPACE}/${slug}`)
}
const { acme, northwind } = DEMO_ORGANIZATIONS
const ada = "ada@example.test"
const grace = "grace@example.test"
const linus = "linus@example.test"
// Acme Review: Ada owns it, Grace works in it, Linus only reads.
await organization(ada, acme)
await member(ada, acme.slug, grace, "member")
await member(ada, acme.slug, linus, "viewer")
await project(ada, acme.slug, "documents", PROJECT.slug, PROJECT.name)
await project(ada, acme.slug, "notes", "notes", "Team notes")
// Northwind: Grace owns it and gave it its own brand; Ada administers it, so she belongs to two
// organizations and the switcher has something to switch between.
await organization(grace, northwind)
await member(grace, northwind.slug, ada, "admin")
await project(grace, northwind.slug, "documents", PROJECT.slug, PROJECT.name)
const branded = (await as(grace).callByName("branding.brand.get", { org: northwind.slug })) as {
displayName: string | null
}
if (branded.displayName === null) {
await as(grace).callByName("branding.brand.set", { org: northwind.slug, ...northwind.brand })
created.push(`brand of ${northwind.slug}`)
}
// A few notes in Acme's notes project, so a module's pages have something on them.
const notes = (await as(ada).callByName("notes.note.list", {
org: acme.slug,
workspace: WORKSPACE,
project: "notes",
})) as { notes: unknown[] }
if (notes.notes.length === 0) {
for (const [title, body] of [
["Welcome", "These notes are demo data. Add one, attach a file, delete it."],
["Roles", "Ada owns this organization, Grace is a member and Linus can only read."],
] as const) {
await as(ada).callByName("notes.note.create", {
org: acme.slug,
workspace: WORKSPACE,
project: "notes",
title,
body,
})
}
created.push("two notes")
}
return {
created,
people: DEMO_PEOPLE,
organizations: [acme, northwind].map(({ slug, name }) => ({ slug, name })),
}
}
+14
View File
@@ -0,0 +1,14 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"noEmit": false,
"declaration": false,
"outDir": "dist",
"rootDir": "src",
"module": "nodenext",
"moduleResolution": "nodenext",
"types": ["node"]
},
"include": ["src/**/*.ts"],
"//": "The emit build resolves a workspace package through node_modules to its dist, not through the source paths of tsconfig.paths.json: pnpm builds the packages in dependency order, so the dist is there."
}
+13
View File
@@ -0,0 +1,13 @@
{
"//": "The worker typechecks against the modules' source (tsconfig.paths.json), and modules/index.ts names the modules' pages, which are React: so the worker program needs the JSX setting and the DOM library to read them, though it never renders one.",
"extends": "../../tsconfig.paths.json",
"compilerOptions": {
"module": "nodenext",
"moduleResolution": "nodenext",
"jsx": "react-jsx",
"lib": ["ES2023", "dom"],
"types": ["node"],
"noEmit": true
},
"include": ["src/**/*.ts"]
}