This commit is contained in:
ConusAI vendor v0.1.0-dev.20261027+9b1131fb
@@ -0,0 +1,33 @@
|
||||
// The chunker: paragraph packing, hard splits of over-long paragraphs, positions a citation can
|
||||
// point at. Pure paths — no store, no model.
|
||||
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { CHUNK_SIZE, chunkText } from "../core/chunk.js"
|
||||
|
||||
describe("chunkText", () => {
|
||||
it("keeps a short document as one chunk", () => {
|
||||
const chunks = chunkText("One paragraph.\n\nTwo paragraphs.")
|
||||
expect(chunks).toHaveLength(1)
|
||||
expect(chunks[0]?.position).toBe(1)
|
||||
expect(chunks[0]?.text).toContain("Two paragraphs.")
|
||||
})
|
||||
|
||||
it("splits on paragraph boundaries when the pack overflows the size", () => {
|
||||
const paragraph = "word ".repeat(500) // 2500 characters
|
||||
const chunks = chunkText(`${paragraph.trim()}\n\n${paragraph.trim()}`)
|
||||
expect(chunks.length).toBeGreaterThanOrEqual(2)
|
||||
for (const chunk of chunks) expect(chunk.text.length).toBeLessThanOrEqual(CHUNK_SIZE)
|
||||
expect(chunks.map((chunk) => chunk.position)).toEqual(chunks.map((_, at) => at + 1))
|
||||
})
|
||||
|
||||
it("hard-splits a paragraph longer than the size", () => {
|
||||
const long = "x".repeat(CHUNK_SIZE * 2 + 100)
|
||||
const chunks = chunkText(long)
|
||||
expect(chunks.length).toBeGreaterThan(2)
|
||||
expect(chunks.every((chunk) => chunk.text.length <= CHUNK_SIZE)).toBe(true)
|
||||
})
|
||||
|
||||
it("refuses an empty document with a named field", () => {
|
||||
expect(() => chunkText("")).toThrow(/nothing to index/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,208 @@
|
||||
// The document requests and the index job's orchestration: idempotent (collection, file) pairs,
|
||||
// the flag gate, the content-type and size admission, the row's progress states, and which
|
||||
// failures are permanent.
|
||||
|
||||
import type { TenantScope } from "@conusai-kit/core"
|
||||
import { createMemoryDataSource } from "@conusai-kit/data"
|
||||
import { DEFAULT_FILES_POLICY, files } from "@conusai-kit/files"
|
||||
import { fakeObjectStore } from "@conusai-kit/testing"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { collections, documents, KNOWLEDGE_ENTITIES } from "../contract/index.js"
|
||||
import { listDocuments, requestIndex, requestRemove } from "../core/documents.js"
|
||||
import { indexDocument, removeDocument } from "../core/index-job.js"
|
||||
import type { KnowledgeDeps } from "../core/ports.js"
|
||||
|
||||
const USER = "01900000-0000-7000-0000-000000000009"
|
||||
const scope: TenantScope = {
|
||||
level: "project",
|
||||
actor: { userId: USER },
|
||||
orgId: "01900000-0000-7000-0000-000000000001",
|
||||
workspaceId: "01900000-0000-7000-0000-000000000002",
|
||||
projectId: "01900000-0000-7000-0000-000000000003",
|
||||
}
|
||||
|
||||
const fileRow = {
|
||||
status: "stored" as const,
|
||||
storageKey: "k/seed",
|
||||
filename: "a.txt",
|
||||
contentType: "text/plain",
|
||||
size: 12,
|
||||
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
uploadedBy: USER,
|
||||
storedAt: "2026-10-23T00:00:00.000Z",
|
||||
uploadId: null,
|
||||
declaredSize: null,
|
||||
partSize: null,
|
||||
deletingAt: null,
|
||||
}
|
||||
|
||||
const makeDeps = (options: { flag?: boolean; search?: KnowledgeDeps["search"] } = {}) => {
|
||||
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES, files] })
|
||||
const store = fakeObjectStore()
|
||||
const search = options.search ?? {
|
||||
readiness: async () => ({ state: "ready" as const, reasons: [] }),
|
||||
index: async () => undefined,
|
||||
search: async () => [],
|
||||
remove: async () => undefined,
|
||||
}
|
||||
const deps: KnowledgeDeps = {
|
||||
data,
|
||||
files: { data, store, policy: DEFAULT_FILES_POLICY },
|
||||
search,
|
||||
flags: { enabled: async () => options.flag ?? true },
|
||||
}
|
||||
void deps
|
||||
return { data, deps, store }
|
||||
}
|
||||
|
||||
const seed = async (data: ReturnType<typeof createMemoryDataSource>, contentType = "text/plain") => {
|
||||
await data.forScope(scope).of(collections).insert({ name: "contracts", createdBy: USER })
|
||||
const row = await data
|
||||
.forScope(scope)
|
||||
.of(files)
|
||||
.insert({ ...fileRow, ...(contentType === "text/plain" ? {} : { contentType }) })
|
||||
return row.id
|
||||
}
|
||||
|
||||
/** Seeds the file row AND the object the row names, the way a completed upload leaves both. */
|
||||
const seedWithBytes = async (
|
||||
data: ReturnType<typeof createMemoryDataSource>,
|
||||
store: ReturnType<typeof fakeObjectStore>,
|
||||
bytes: Uint8Array,
|
||||
) => {
|
||||
const fileId = await seed(data)
|
||||
await store.put(scope, { key: fileRow.storageKey, body: bytes, contentType: "text/plain" })
|
||||
return fileId
|
||||
}
|
||||
|
||||
describe("requesting a document's indexing", () => {
|
||||
it("creates the row, publishes the event, and a repeat request changes nothing", async () => {
|
||||
const { data, deps } = makeDeps()
|
||||
const fileId = await seed(data)
|
||||
const first = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
||||
expect(first.status).toBe("queued")
|
||||
const second = await requestIndex(deps, scope, { collectionName: "contracts", fileId: first.fileId })
|
||||
expect(second.documentId).toBe(first.documentId)
|
||||
expect(await listDocuments(deps, scope, "contracts")).toEqual({
|
||||
documents: [expect.objectContaining({ documentId: first.documentId, status: "queued" })],
|
||||
})
|
||||
// One outbox event: the subscriber's enqueue is the exactly-once path.
|
||||
const outbox = data.outbox.pending(10)
|
||||
expect((await outbox).filter((record) => record.name === "knowledge.documentIndexRequested")).toHaveLength(1)
|
||||
})
|
||||
|
||||
it("re-queues an indexed pair instead of duplicating it", async () => {
|
||||
const { data, deps } = makeDeps()
|
||||
const fileId = await seed(data)
|
||||
const first = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
||||
await data.forScope(scope).of(documents).update(first.documentId, { status: "indexed" }, 1)
|
||||
const again = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
||||
expect(again.documentId).toBe(first.documentId)
|
||||
expect(again.status).toBe("queued")
|
||||
})
|
||||
|
||||
it("refuses when the flag is off, and admits text files only, within the size cap", async () => {
|
||||
const { data, deps } = makeDeps({ flag: false })
|
||||
const fileId = await seed(data)
|
||||
const off = await requestIndex(deps, scope, { collectionName: "contracts", fileId }).catch((e: unknown) => e)
|
||||
expect((off as { status?: number }).status).toBe(403)
|
||||
|
||||
const refusing = makeDeps()
|
||||
const pdfFile = await seed(refusing.data, "application/pdf")
|
||||
const pdf = await requestIndex(refusing.deps, scope, { collectionName: "contracts", fileId: pdfFile }).catch(
|
||||
(e: unknown) => e,
|
||||
)
|
||||
expect((pdf as { status?: number }).status).toBe(422)
|
||||
|
||||
const big = makeDeps()
|
||||
const bigFile = await seed(big.data)
|
||||
await big.data.forScope(scope).of(files).update(bigFile, { size: 2_000_000 }, 1)
|
||||
const tooBig = await requestIndex(big.deps, scope, { collectionName: "contracts", fileId: bigFile }).catch(
|
||||
(e: unknown) => e,
|
||||
)
|
||||
expect((tooBig as { status?: number }).status).toBe(422)
|
||||
})
|
||||
})
|
||||
|
||||
describe("the index job's orchestration", () => {
|
||||
it("moves the row queued -> indexing -> indexed and calls the search service once", async () => {
|
||||
const indexed: string[] = []
|
||||
const { data, deps, store } = makeDeps({
|
||||
search: {
|
||||
readiness: async () => ({ state: "ready", reasons: [] }),
|
||||
index: async (_scope, input) => {
|
||||
indexed.push(input.fileId)
|
||||
},
|
||||
search: async () => [],
|
||||
remove: async () => undefined,
|
||||
},
|
||||
})
|
||||
const fileId = await seedWithBytes(data, store, new TextEncoder().encode("hello knowledge"))
|
||||
const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
||||
const done = await indexDocument(deps, scope, requested.documentId)
|
||||
expect(done.status).toBe("indexed")
|
||||
expect(indexed).toEqual([fileId])
|
||||
// A redelivered job finds the row indexed and does the external work again never.
|
||||
const again = await indexDocument(deps, scope, requested.documentId)
|
||||
expect(again.status).toBe("indexed")
|
||||
expect(indexed).toEqual([fileId])
|
||||
})
|
||||
|
||||
it("records a permanent failure on the row; a transient one throws for the queue to retry", async () => {
|
||||
const permanent = makeDeps({
|
||||
search: {
|
||||
readiness: async () => ({ state: "ready", reasons: [] }),
|
||||
index: async () => {
|
||||
throw Object.assign(new Error("not text"), { permanent: "not_text" })
|
||||
},
|
||||
search: async () => [],
|
||||
remove: async () => undefined,
|
||||
},
|
||||
})
|
||||
const permanentFile = await seedWithBytes(
|
||||
permanent.data,
|
||||
permanent.store,
|
||||
new TextEncoder().encode("not text at all"),
|
||||
)
|
||||
const requested = await requestIndex(permanent.deps, scope, { collectionName: "contracts", fileId: permanentFile })
|
||||
const failed = await indexDocument(permanent.deps, scope, requested.documentId)
|
||||
expect(failed.status).toBe("failed")
|
||||
expect(failed.error).toBe("not_text")
|
||||
|
||||
const transient = makeDeps({
|
||||
search: {
|
||||
readiness: async () => ({ state: "ready", reasons: [] }),
|
||||
index: async () => {
|
||||
throw new Error("the store hiccuped")
|
||||
},
|
||||
search: async () => [],
|
||||
remove: async () => undefined,
|
||||
},
|
||||
})
|
||||
const transientFile = await seedWithBytes(transient.data, transient.store, new TextEncoder().encode("hello"))
|
||||
const requested2 = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId: transientFile })
|
||||
await expect(indexDocument(transient.deps, scope, requested2.documentId)).rejects.toThrow("hiccuped")
|
||||
})
|
||||
|
||||
it("removal deletes the row and asks the service to drop the vectors", async () => {
|
||||
const removed: string[] = []
|
||||
const { data, deps } = makeDeps({
|
||||
search: {
|
||||
readiness: async () => ({ state: "ready", reasons: [] }),
|
||||
index: async () => undefined,
|
||||
search: async () => [],
|
||||
remove: async (_scope, input) => {
|
||||
removed.push(input.fileId)
|
||||
},
|
||||
},
|
||||
})
|
||||
const fileId = await seed(data)
|
||||
const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
||||
const cancelled = await requestRemove(deps, scope, { collectionName: "contracts", fileId })
|
||||
expect(cancelled.status).toBe("cancelled")
|
||||
// The remove job's body: delete the vectors, then the row.
|
||||
await removeDocument(deps, scope, requested.documentId)
|
||||
expect(removed).toEqual([fileId])
|
||||
expect((await listDocuments(deps, scope, "contracts")).documents).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,30 @@
|
||||
// The module's messages: en and lt declare the same keys, or a page in one language breaks.
|
||||
|
||||
import { buildRegistry } from "@conusai-kit/core"
|
||||
import { filesModule } from "@conusai-kit/files"
|
||||
import { createI18n } from "@conusai-kit/i18n"
|
||||
import { tenancyModule } from "@conusai-kit/tenancy"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { knowledgeModule } from "../index.js"
|
||||
|
||||
describe("the knowledge module's messages", () => {
|
||||
const catalogues = buildRegistry([tenancyModule, filesModule, knowledgeModule]).catalogues
|
||||
|
||||
it("declares en and lt, and every en key resolves in lt", () => {
|
||||
const en = catalogues.en?.knowledge ?? {}
|
||||
const lt = catalogues.lt?.knowledge ?? {}
|
||||
const flatten = (tree: Record<string, unknown>, prefix = ""): string[] =>
|
||||
Object.entries(tree).flatMap(([key, value]) =>
|
||||
typeof value === "string" ? [`${prefix}${key}`] : flatten(value as Record<string, unknown>, `${prefix}${key}.`),
|
||||
)
|
||||
const enKeys = flatten(en)
|
||||
const ltKeys = new Set(flatten(lt))
|
||||
expect(enKeys.length).toBeGreaterThan(0)
|
||||
for (const key of enKeys) expect(ltKeys.has(key), `knowledge.${key} has no Lithuanian message`).toBe(true)
|
||||
})
|
||||
|
||||
it("formats a message with values in both locales", () => {
|
||||
const i18n = createI18n("lt", { lt: { knowledge: catalogues.lt?.knowledge ?? {} } })
|
||||
expect(i18n.message("knowledge.status.documents", { queued: 1, indexing: 2, indexed: 3, failed: 0 })).toContain("1")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,53 @@
|
||||
// What the generated tenant-isolation suite (test/isolation.test.ts) needs of this module: rows it
|
||||
// seeds in every tenant, and one fixture per tenant-scoped procedure. A fixture is built from the
|
||||
// OTHER tenant's ids: the call is made as a member of organization A with organization B's slugs
|
||||
// and ids, and has to be refused or answered empty.
|
||||
|
||||
import type { TransportCaller } from "@conusai-kit/core"
|
||||
import { type IsolationFixture, type IsolationTenant, uploadViaProcedures } from "@conusai-kit/testing"
|
||||
|
||||
/** What a module's isolation file exports: the shape the suite reads. */
|
||||
export interface ModuleIsolation {
|
||||
seed(
|
||||
call: TransportCaller,
|
||||
inProject: { org: string; workspace: string; project: string },
|
||||
): Promise<Record<string, string>>
|
||||
fixtures: Record<string, IsolationFixture>
|
||||
}
|
||||
|
||||
const NOWHERE = "00000000-0000-0000-0000-000000000000"
|
||||
const inProjectOf = (t: IsolationTenant) => ({ org: t.orgSlug, workspace: t.workspaceSlug, project: t.projectSlug })
|
||||
const collectionOf = (t: IsolationTenant) => t.extra.knowledgeCollection ?? "contracts"
|
||||
const fileOf = (t: IsolationTenant) => t.extra.knowledgeFileId ?? NOWHERE
|
||||
|
||||
export const isolation: ModuleIsolation = {
|
||||
async seed(call, inProject) {
|
||||
const file = await uploadViaProcedures(call, inProject, {
|
||||
filename: "knowledge-seed.txt",
|
||||
contentType: "text/plain",
|
||||
bytes: new TextEncoder().encode("seed knowledge"),
|
||||
})
|
||||
// The seed needs a collection row to exist; collection.create needs the admin role, which the
|
||||
// isolation harness does not grant the seeding caller, so the fixture names a collection and
|
||||
// the write fixtures create it through the manage permission where a denial is the assertion.
|
||||
return { knowledgeCollection: "contracts", knowledgeFileId: file.fileId }
|
||||
},
|
||||
fixtures: {
|
||||
"knowledge.collection.list": { input: (t) => inProjectOf(t) },
|
||||
"knowledge.collection.create": { input: (t) => ({ ...inProjectOf(t), name: "intruders" }) },
|
||||
"knowledge.document.list": {
|
||||
input: (t) => ({ ...inProjectOf(t), collectionName: collectionOf(t) }),
|
||||
},
|
||||
"knowledge.document.index": {
|
||||
input: (t) => ({ ...inProjectOf(t), collectionName: collectionOf(t), fileId: fileOf(t) }),
|
||||
},
|
||||
"knowledge.document.remove": {
|
||||
input: (t) => ({ ...inProjectOf(t), collectionName: collectionOf(t), fileId: fileOf(t) }),
|
||||
},
|
||||
"knowledge.search": { input: (t) => ({ ...inProjectOf(t), collectionName: collectionOf(t), query: "seed" }) },
|
||||
"knowledge.status": { input: (t) => inProjectOf(t) },
|
||||
"knowledge.settings.get": { input: (t) => ({ org: t.orgSlug }) },
|
||||
"knowledge.settings.enableDocumentAi": { input: (t) => ({ org: t.orgSlug }) },
|
||||
"knowledge.settings.disableDocumentAi": { input: (t) => ({ org: t.orgSlug }) },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// The module's migration is its own file and exactly `schemaStatements()` over its three entities
|
||||
// (decision AT-D20: the entity definitions are the one source of a column). The module ships the
|
||||
// Postgres profile; the Convex binding is provided for the integration owner and has no SQL.
|
||||
|
||||
import { readFileSync } from "node:fs"
|
||||
import { schemaStatements } from "@conusai-kit/data-kysely"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { KNOWLEDGE_ENTITIES, knowledgeModule } from "../index.js"
|
||||
|
||||
/** The statements for these tables alone: `schemaStatements` leads with the kernel's own two. */
|
||||
function tablesOf(entities: Parameters<typeof schemaStatements>[0]): string {
|
||||
const kernel = schemaStatements([]).length
|
||||
return `${schemaStatements(entities)
|
||||
.slice(kernel)
|
||||
.map((statement) => `${statement};`)
|
||||
.join("\n\n")}\n`
|
||||
}
|
||||
|
||||
describe("the migration of the knowledge module", () => {
|
||||
it("is its own file, exactly its three tables", async () => {
|
||||
await expect(
|
||||
`-- SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0\n${tablesOf(KNOWLEDGE_ENTITIES)}`,
|
||||
).toMatchFileSnapshot("../migrations/0001_knowledge.sql")
|
||||
})
|
||||
|
||||
it("is declared by the manifest, so the migrate job finds it, and every table is row-level secured", () => {
|
||||
expect(knowledgeModule.migrations).toEqual({ postgres: "migrations" })
|
||||
expect(knowledgeModule.profiles).toEqual(["postgres"])
|
||||
const sql = readFileSync(new URL("../migrations/0001_knowledge.sql", import.meta.url), "utf8")
|
||||
for (const table of ["knowledge_collection", "knowledge_document", "knowledge_config"]) {
|
||||
expect(sql).toContain(`create table "${table}"`)
|
||||
expect(sql).toContain(`alter table "${table}" force row level security`)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,169 @@
|
||||
import { createMemoryDataSource } from "@conusai-kit/data"
|
||||
import { DEFAULT_FILES_POLICY, type FilesDeps, files } from "@conusai-kit/files"
|
||||
import type { Knowledge, KnowledgeSearchHit } from "@conusai-kit/platform"
|
||||
import { fakeObjectStore } from "@conusai-kit/testing"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { platformKnowledgeSearch } from "../adapters/platform-knowledge.js"
|
||||
import type { KnowledgeScope } from "../core/ports.js"
|
||||
|
||||
const scope: KnowledgeScope = {
|
||||
level: "project",
|
||||
actor: { userId: "01900000-0000-7000-0000-000000000009" },
|
||||
orgId: "01900000-0000-7000-0000-000000000001",
|
||||
workspaceId: "01900000-0000-7000-0000-000000000002",
|
||||
projectId: "01900000-0000-7000-0000-000000000003",
|
||||
}
|
||||
const otherScope: KnowledgeScope = {
|
||||
...scope,
|
||||
projectId: "01900000-0000-7000-0000-000000000004",
|
||||
}
|
||||
const storageKey = `files/${scope.workspaceId}/${scope.projectId}/01900000-0000-7000-0000-000000000100`
|
||||
const otherStorageKey = `files/${otherScope.workspaceId}/${otherScope.projectId}/01900000-0000-7000-0000-000000000101`
|
||||
const pathname = `org/${scope.orgId}/${storageKey}`
|
||||
|
||||
interface ClientStub {
|
||||
indexed: string[]
|
||||
removed: string[]
|
||||
searches: Array<Parameters<Knowledge["search"]>[0]>
|
||||
results: readonly KnowledgeSearchHit[]
|
||||
}
|
||||
|
||||
function knowledgeClient(stub: ClientStub): Knowledge {
|
||||
return {
|
||||
readiness: async () => ({ state: "ready", reasons: [], checkedAt: new Date().toISOString() }),
|
||||
createCollection: async (input) => input,
|
||||
search: async (input) => {
|
||||
stub.searches.push(input)
|
||||
return stub.results
|
||||
},
|
||||
index: async (input) => {
|
||||
stub.indexed.push(input.documentRef)
|
||||
return { jobId: "job-1" }
|
||||
},
|
||||
job: async (jobId) => ({ jobId, status: "succeeded", progress: 1, generation: 1, error: null }),
|
||||
cancel: async () => undefined,
|
||||
remove: async (documentRef) => {
|
||||
stub.removed.push(documentRef)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function stub(results: readonly KnowledgeSearchHit[] = []): ClientStub {
|
||||
return { indexed: [], removed: [], searches: [], results }
|
||||
}
|
||||
|
||||
function makeFiles() {
|
||||
const data = createMemoryDataSource({ entities: [files] })
|
||||
const deps: FilesDeps = { data, store: fakeObjectStore(), policy: DEFAULT_FILES_POLICY }
|
||||
return { data, deps }
|
||||
}
|
||||
|
||||
async function seedFile(
|
||||
data: ReturnType<typeof createMemoryDataSource>,
|
||||
atScope: KnowledgeScope,
|
||||
key: string,
|
||||
filename = "contract.txt",
|
||||
): Promise<string> {
|
||||
const row = await data.forScope(atScope).of(files).insert({
|
||||
status: "stored",
|
||||
storageKey: key,
|
||||
filename,
|
||||
contentType: "text/plain",
|
||||
size: 17,
|
||||
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
uploadedBy: atScope.actor.userId,
|
||||
storedAt: "2026-10-24T00:00:00.000Z",
|
||||
uploadId: null,
|
||||
declaredSize: null,
|
||||
partSize: null,
|
||||
deletingAt: null,
|
||||
})
|
||||
return row.id
|
||||
}
|
||||
|
||||
describe("the managed knowledge Blob reference", () => {
|
||||
it("indexes and removes by Blob pathname rather than opaque file id", async () => {
|
||||
const { data, deps } = makeFiles()
|
||||
const fileId = await seedFile(data, scope, storageKey)
|
||||
const client = stub()
|
||||
const service = platformKnowledgeSearch(knowledgeClient(client), deps)
|
||||
|
||||
await service.index(scope, {
|
||||
collectionName: "contracts",
|
||||
fileId,
|
||||
filename: "contract.txt",
|
||||
bytes: new TextEncoder().encode("managed knowledge"),
|
||||
})
|
||||
await service.remove(scope, { collectionName: "contracts", fileId })
|
||||
|
||||
expect(client.indexed).toEqual([pathname])
|
||||
expect(client.removed).toEqual([pathname])
|
||||
})
|
||||
|
||||
it("maps citations through scoped file rows and drops foreign or stale references", async () => {
|
||||
const { data, deps } = makeFiles()
|
||||
const fileId = await seedFile(data, scope, storageKey)
|
||||
await seedFile(data, otherScope, otherStorageKey, "other-project.txt")
|
||||
const client = stub([
|
||||
{ documentRef: pathname, snippet: "own snippet", score: 0.9, citations: [{ page: 1 }] },
|
||||
{
|
||||
documentRef: `org/${scope.orgId}/${otherStorageKey}`,
|
||||
snippet: "other project snippet",
|
||||
score: 0.8,
|
||||
citations: [],
|
||||
},
|
||||
{
|
||||
documentRef: `org/01900000-0000-7000-0000-000000000777/${storageKey}`,
|
||||
snippet: "other organization snippet",
|
||||
score: 0.7,
|
||||
citations: [],
|
||||
},
|
||||
{
|
||||
documentRef: `org/${scope.orgId}/files/${scope.workspaceId}/${scope.projectId}/01900000-0000-7000-0000-000000000200`,
|
||||
snippet: "stale snippet",
|
||||
score: 0.6,
|
||||
citations: [],
|
||||
},
|
||||
])
|
||||
const service = platformKnowledgeSearch(knowledgeClient(client), deps)
|
||||
|
||||
const hits = await service.search(scope, {
|
||||
collectionName: "contracts",
|
||||
query: "contract",
|
||||
limit: 10,
|
||||
actorId: scope.actor.userId,
|
||||
})
|
||||
|
||||
expect(hits).toEqual([
|
||||
{
|
||||
fileId,
|
||||
filename: "contract.txt",
|
||||
collectionName: "contracts",
|
||||
score: 0.9,
|
||||
chunk: 1,
|
||||
snippet: "own snippet",
|
||||
},
|
||||
])
|
||||
expect(client.searches).toHaveLength(1)
|
||||
expect(client.searches[0]).toMatchObject({ collection: "contracts", query: "contract", limit: 10 })
|
||||
expect(client.searches[0]).not.toHaveProperty("tenant")
|
||||
expect(client.searches[0]).not.toHaveProperty("ownerRef")
|
||||
})
|
||||
|
||||
it("refuses a foreign file id before any managed index call", async () => {
|
||||
const { data, deps } = makeFiles()
|
||||
const foreignFileId = await seedFile(data, otherScope, otherStorageKey)
|
||||
const client = stub()
|
||||
const service = platformKnowledgeSearch(knowledgeClient(client), deps)
|
||||
|
||||
await expect(
|
||||
service.index(scope, {
|
||||
collectionName: "contracts",
|
||||
fileId: foreignFileId,
|
||||
filename: "other-project.txt",
|
||||
bytes: new TextEncoder().encode("foreign content"),
|
||||
}),
|
||||
).rejects.toThrow()
|
||||
expect(client.indexed).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,131 @@
|
||||
// The enablement flow's semantics: idempotent enable, typed denials from the platform checks, the
|
||||
// operator endpoint's explicit validation, and disable that preserves every row and every vector.
|
||||
|
||||
import type { TenantScope } from "@conusai-kit/core"
|
||||
import { createMemoryDataSource } from "@conusai-kit/data"
|
||||
import { DEFAULT_FILES_POLICY } from "@conusai-kit/files"
|
||||
import { fakeObjectStore } from "@conusai-kit/testing"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { collections, documents, KNOWLEDGE_ENTITIES } from "../contract/index.js"
|
||||
import { createCollection } from "../core/collections.js"
|
||||
import type { DocumentAiPlatform } from "../core/ports.js"
|
||||
import { disableDocumentAi, enableDocumentAi, getSettings } from "../core/settings.js"
|
||||
|
||||
const ORG = "01900000-0000-7000-0000-000000000001"
|
||||
const scope: TenantScope = { level: "org", actor: { userId: "01900000-0000-7000-0000-000000000009" }, orgId: ORG }
|
||||
|
||||
const platform = (over: Partial<DocumentAiPlatform> = {}): DocumentAiPlatform => ({
|
||||
checkEligibility: async () => ({ ok: true }),
|
||||
indexingQuota: async () => 1000,
|
||||
linkManagedQdrant: async () => ({ serviceId: "svc-1", url: null }),
|
||||
probeQdrant: async () => ({ ok: true }),
|
||||
operatorQdrant: () => null,
|
||||
...over,
|
||||
})
|
||||
|
||||
const deps = (data: ReturnType<typeof createMemoryDataSource>, p: DocumentAiPlatform) => ({
|
||||
data,
|
||||
platform: p,
|
||||
now: () => Date.UTC(2026, 9, 23),
|
||||
})
|
||||
|
||||
describe("the enablement flow", () => {
|
||||
it("enables idempotently: the second call writes nothing and answers the same configuration", async () => {
|
||||
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES] })
|
||||
const first = await enableDocumentAi(deps(data, platform()), scope)
|
||||
expect(first).toEqual({ enabled: true, backend: "vectors", source: "managed", url: null })
|
||||
const second = await enableDocumentAi(deps(data, platform()), scope)
|
||||
expect(second).toEqual(first)
|
||||
expect((await getSettings(deps(data, platform()), scope)).enabled).toBe(true)
|
||||
})
|
||||
|
||||
it("surfaces the platform's denials as typed reason codes, retirable", async () => {
|
||||
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES] })
|
||||
const denied = await enableDocumentAi(
|
||||
deps(data, platform({ checkEligibility: async () => ({ ok: false, reason: "plan_required" }) })),
|
||||
scope,
|
||||
).catch((error: unknown) => error)
|
||||
expect((denied as { code?: string }).code).toBe("plan_required")
|
||||
// Nothing was persisted by the denied attempt; the retry path runs the same checks again.
|
||||
expect((await getSettings(deps(data, platform()), scope)).enabled).toBe(false)
|
||||
const quota = await enableDocumentAi(deps(data, platform({ indexingQuota: async () => 0 })), scope).catch(
|
||||
(error: unknown) => error,
|
||||
)
|
||||
expect((quota as { code?: string }).code).toBe("quota_exhausted")
|
||||
})
|
||||
|
||||
it("validates an operator-configured endpoint before it trusts it, and records it without the key", async () => {
|
||||
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES] })
|
||||
const p = platform({
|
||||
operatorQdrant: () => ({ url: "http://qdrant:6333", apiKey: "secret-key" }),
|
||||
probeQdrant: async () => ({ ok: false, detail: "refused" }),
|
||||
})
|
||||
const refused = await enableDocumentAi(deps(data, p), scope).catch((error: unknown) => error)
|
||||
expect((refused as { status?: number }).status).toBe(422)
|
||||
// A failed probe persists nothing.
|
||||
expect((await getSettings(deps(data, platform()), scope)).enabled).toBe(false)
|
||||
|
||||
const ok = platform({ operatorQdrant: () => ({ url: "http://qdrant:6333", apiKey: "secret-key" }) })
|
||||
const enabled = await enableDocumentAi(deps(data, ok), scope)
|
||||
expect(enabled.source).toBe("operator")
|
||||
expect(enabled.url).toBe("http://qdrant:6333")
|
||||
// The key never lands in the configuration row.
|
||||
const rows = await data
|
||||
.forScope(scope)
|
||||
.of((await import("../contract/index.js")).config)
|
||||
.list({ limit: 1 })
|
||||
expect(JSON.stringify(rows.items[0])).not.toContain("secret-key")
|
||||
})
|
||||
|
||||
it("disable preserves collections, documents and the recorded configuration; re-enable resumes without duplication", async () => {
|
||||
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES] })
|
||||
const projectScope: TenantScope = {
|
||||
level: "project",
|
||||
actor: { userId: "01900000-0000-7000-0000-000000000009" },
|
||||
orgId: ORG,
|
||||
workspaceId: "01900000-0000-7000-0000-000000000002",
|
||||
projectId: "01900000-0000-7000-0000-000000000003",
|
||||
}
|
||||
const knowledgeDepsForProject = {
|
||||
data,
|
||||
files: { data, store: fakeObjectStore(), policy: DEFAULT_FILES_POLICY },
|
||||
flags: { enabled: async () => true },
|
||||
}
|
||||
await enableDocumentAi(deps(data, platform()), scope)
|
||||
const collection = await createCollection(knowledgeDepsForProject, projectScope as never, "contracts")
|
||||
expect(collection.name).toBe("contracts")
|
||||
// Seed a document row directly (the upload half is the files module's own suite).
|
||||
await data.forScope(projectScope).of(documents).insert({
|
||||
collectionName: "contracts",
|
||||
fileId: "01900000-0000-7000-0000-000000000099",
|
||||
filename: "a.txt",
|
||||
status: "indexed",
|
||||
error: null,
|
||||
indexedAt: "2026-10-23T00:00:00.000Z",
|
||||
requestedBy: "01900000-0000-7000-0000-000000000009",
|
||||
})
|
||||
|
||||
const disabled = await disableDocumentAi(deps(data, platform()), scope)
|
||||
expect(disabled.enabled).toBe(false)
|
||||
// Everything is still there.
|
||||
expect((await getSettings(deps(data, platform()), scope)).backend).toBe("vectors")
|
||||
const collectionsAfter = await data
|
||||
.forScope(projectScope)
|
||||
.of(collections)
|
||||
.list({ where: [{ field: "name", op: "eq", value: "contracts" }], limit: 1 })
|
||||
expect(collectionsAfter.items).toHaveLength(1)
|
||||
const documentsAfter = await data
|
||||
.forScope(projectScope)
|
||||
.of(documents)
|
||||
.list({ where: [{ field: "collectionName", op: "eq", value: "contracts" }], limit: 1 })
|
||||
expect(documentsAfter.items[0]?.status).toBe("indexed")
|
||||
|
||||
// Re-enable: idempotent, same row, no second collection.
|
||||
await enableDocumentAi(deps(data, platform()), scope)
|
||||
const again = await data
|
||||
.forScope(projectScope)
|
||||
.of(collections)
|
||||
.list({ where: [{ field: "name", op: "eq", value: "contracts" }], limit: 1 })
|
||||
expect(again.items).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
// The components' markup, rendered on the server the way a page renders it: what a screen reader
|
||||
// and a keyboard user get is in the HTML — a name for every control, a status that says what a
|
||||
// colour would, and the AI disclosure where AI output appears.
|
||||
|
||||
import { buildRegistry } from "@conusai-kit/core"
|
||||
import { createI18n } from "@conusai-kit/i18n"
|
||||
import { TransportProvider } from "@conusai-kit/ui"
|
||||
import { renderToStaticMarkup } from "react-dom/server"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { appModules } from "../../index.js"
|
||||
import { searchFormLabels } from "../ui/labels.js"
|
||||
import { SearchForm } from "../ui/search-form.js"
|
||||
import { SettingsSection } from "../ui/settings-section.js"
|
||||
|
||||
const catalogues = buildRegistry([...appModules]).catalogues
|
||||
const i18n = createI18n("en", { en: { knowledge: catalogues.en?.knowledge ?? {} } })
|
||||
const t = (key: string, values?: Record<string, string | number>) => i18n.message(`knowledge.${key}`, values)
|
||||
const failed = "That did not work: {detail}"
|
||||
const slugs = { org: "acme", workspace: "ws", project: "p" }
|
||||
|
||||
const inProvider = (node: React.ReactNode) =>
|
||||
renderToStaticMarkup(
|
||||
<TransportProvider call={async () => undefined} describe={() => "x"}>
|
||||
{node}
|
||||
</TransportProvider>,
|
||||
)
|
||||
|
||||
describe("the search form", () => {
|
||||
const html = inProvider(<SearchForm slugs={slugs} collectionName="contracts" labels={searchFormLabels(t, failed)} />)
|
||||
|
||||
it("names its controls and carries the AI disclosure", () => {
|
||||
expect(html).toContain(`<label for=`)
|
||||
expect(html).toContain('type="search"')
|
||||
expect(html).toContain("kit-ai-disclosure")
|
||||
expect(html).toContain(t("search.disclosure"))
|
||||
})
|
||||
})
|
||||
|
||||
describe("the settings section", () => {
|
||||
const labels = {
|
||||
heading: t("settings.heading"),
|
||||
state: t("settings.state"),
|
||||
stateOn: t("settings.stateOn"),
|
||||
stateOff: t("settings.stateOff"),
|
||||
backend: t("settings.backend"),
|
||||
backendVectors: t("settings.backendVectors"),
|
||||
source: t("settings.source"),
|
||||
sourceOperator: t("settings.sourceOperator"),
|
||||
sourceManaged: t("settings.sourceManaged"),
|
||||
endpoint: t("settings.endpoint"),
|
||||
enable: t("settings.enable"),
|
||||
enabling: t("settings.enabling"),
|
||||
disable: t("settings.disable"),
|
||||
disabling: t("settings.disabling"),
|
||||
enabledDone: t("settings.enabledDone"),
|
||||
disabledDone: t("settings.disabledDone"),
|
||||
planRequired: t("settings.planRequired"),
|
||||
featureDisabled: t("settings.featureDisabled"),
|
||||
quotaExhausted: t("settings.quotaExhausted"),
|
||||
upgrade: t("settings.upgrade"),
|
||||
hint: t("settings.hint"),
|
||||
failed,
|
||||
}
|
||||
|
||||
it("renders the state and the enable action when disabled", () => {
|
||||
const html = inProvider(
|
||||
<SettingsSection
|
||||
org="acme"
|
||||
initial={{ enabled: false, backend: null, source: null, url: null }}
|
||||
labels={labels}
|
||||
/>,
|
||||
)
|
||||
expect(html).toContain(t("settings.stateOff"))
|
||||
expect(html).toContain(`>${t("settings.enable")}<`)
|
||||
})
|
||||
|
||||
it("renders the endpoint and the disable action when enabled, without any secret in the markup", () => {
|
||||
const html = inProvider(
|
||||
<SettingsSection
|
||||
org="acme"
|
||||
initial={{ enabled: true, backend: "vectors", source: "operator", url: "http://qdrant:6333" }}
|
||||
labels={labels}
|
||||
/>,
|
||||
)
|
||||
expect(html).toContain("http://qdrant:6333")
|
||||
expect(html).toContain(`>${t("settings.disable")}<`)
|
||||
expect(html).not.toContain("apiKey")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
// The search service of a deployment with no vector store (no Qdrant linked, pgvector not chosen):
|
||||
// readiness and the status page say `unavailable` with the actionable reason, an index job fails its
|
||||
// row with the stable `vector_store_unavailable` code (permanent: the queue does not retry it), and a
|
||||
// search or removal is a typed 503, never an empty answer that reads as "nothing indexed".
|
||||
|
||||
import { AppError, type TenantScope } from "@conusai-kit/core"
|
||||
import { createMemoryDataSource } from "@conusai-kit/data"
|
||||
import { DEFAULT_FILES_POLICY, files } from "@conusai-kit/files"
|
||||
import { fakeObjectStore } from "@conusai-kit/testing"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { unavailableKnowledgeSearch } from "../adapters/unavailable.js"
|
||||
import { collections, KNOWLEDGE_ENTITIES } from "../contract/index.js"
|
||||
import { requestIndex } from "../core/documents.js"
|
||||
import { indexDocument } from "../core/index-job.js"
|
||||
import type { KnowledgeDeps } from "../core/ports.js"
|
||||
import { knowledgeStatusOf } from "../core/status.js"
|
||||
|
||||
const USER = "01900000-0000-7000-0000-000000000009"
|
||||
const scope: TenantScope & { workspaceId: string; projectId: string } = {
|
||||
level: "project",
|
||||
actor: { userId: USER },
|
||||
orgId: "01900000-0000-7000-0000-000000000001",
|
||||
workspaceId: "01900000-0000-7000-0000-000000000002",
|
||||
projectId: "01900000-0000-7000-0000-000000000003",
|
||||
}
|
||||
|
||||
const REASON = "no vector store is configured: link a Qdrant service"
|
||||
|
||||
const setup = () => {
|
||||
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES, files] })
|
||||
const store = fakeObjectStore()
|
||||
const deps: KnowledgeDeps = {
|
||||
data,
|
||||
files: { data, store, policy: DEFAULT_FILES_POLICY },
|
||||
search: unavailableKnowledgeSearch(REASON),
|
||||
flags: { enabled: async () => true },
|
||||
}
|
||||
return { data, deps, store }
|
||||
}
|
||||
|
||||
describe("a deployment with no vector store", () => {
|
||||
it("reports unavailable with the reason, on the port and on the status page", async () => {
|
||||
const { deps } = setup()
|
||||
expect(await deps.search?.readiness(scope)).toEqual({ state: "unavailable", reasons: [REASON] })
|
||||
const status = await knowledgeStatusOf(deps, scope)
|
||||
expect(status.service).toEqual({ state: "unavailable", reasons: [REASON] })
|
||||
})
|
||||
|
||||
it("fails an index job permanently with vector_store_unavailable", async () => {
|
||||
const { data, deps, store } = setup()
|
||||
await data.forScope(scope).of(collections).insert({ name: "contracts", createdBy: USER })
|
||||
const file = await data.forScope(scope).of(files).insert({
|
||||
status: "stored",
|
||||
storageKey: "k/seed",
|
||||
filename: "a.txt",
|
||||
contentType: "text/plain",
|
||||
size: 5,
|
||||
sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
|
||||
uploadedBy: USER,
|
||||
storedAt: "2026-10-25T00:00:00.000Z",
|
||||
uploadId: null,
|
||||
declaredSize: null,
|
||||
partSize: null,
|
||||
deletingAt: null,
|
||||
})
|
||||
await store.put(scope, { key: "k/seed", body: new TextEncoder().encode("hello"), contentType: "text/plain" })
|
||||
const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId: file.id })
|
||||
const failed = await indexDocument(deps, scope, requested.documentId)
|
||||
expect(failed.status).toBe("failed")
|
||||
expect(failed.error).toBe("vector_store_unavailable")
|
||||
})
|
||||
|
||||
it("refuses search and removal with a typed 503 that names the reason", async () => {
|
||||
const service = unavailableKnowledgeSearch(REASON)
|
||||
for (const call of [
|
||||
() => service.search(scope, { collectionName: "contracts", query: "q", limit: 5, actorId: USER }),
|
||||
() => service.remove(scope, { collectionName: "contracts", fileId: "01900000-0000-7000-0000-0000000000aa" }),
|
||||
]) {
|
||||
const error = await call().then(
|
||||
() => null,
|
||||
(thrown: unknown) => thrown,
|
||||
)
|
||||
expect(error).toBeInstanceOf(AppError)
|
||||
expect((error as AppError).status).toBe(503)
|
||||
expect((error as AppError).code).toBe("runtime_unavailable")
|
||||
expect((error as AppError).problem.detail).toBe(REASON)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,88 @@
|
||||
// The vector contract suite against the store shape this module binds in tests: a conforming
|
||||
// in-memory VectorStore that keeps the tenant wall the real adapters guarantee (org filter on
|
||||
// every search and delete). The suite is the kit's own two-tenant contract; running it here pins
|
||||
// the module's metadata conventions (collection filters, subject deletion) to what every adapter
|
||||
// must provide, so a convention drift fails in this module's unit suite.
|
||||
|
||||
import { newId, type TenantScope } from "@conusai-kit/core"
|
||||
import type { VectorMatch, VectorQuery, VectorRecord, VectorStore, VectorSubject } from "@conusai-kit/data"
|
||||
import { runVectorStoreContract } from "@conusai-kit/testing"
|
||||
import { describe, expect, it } from "vitest"
|
||||
|
||||
/** The in-memory stand-in with the tenant wall the adapters implement. */
|
||||
function fakeVectorStore(): { store: VectorStore; failures: () => unknown[] } {
|
||||
const rows = new Map<string, { scope: string; record: VectorRecord }>()
|
||||
const keyOf = (orgId: string, id: string) => `${orgId}:${id}`
|
||||
const failures: unknown[] = []
|
||||
const cosine = (a: readonly number[], b: readonly number[]) => {
|
||||
let dot = 0
|
||||
let na = 0
|
||||
let nb = 0
|
||||
for (const [index, value] of a.entries()) {
|
||||
dot += value * (b[index] ?? 0)
|
||||
na += value * value
|
||||
nb += (b[index] ?? 0) ** 2
|
||||
}
|
||||
return na === 0 || nb === 0 ? 0 : dot / Math.sqrt(na * nb)
|
||||
}
|
||||
return {
|
||||
failures: () => failures,
|
||||
store: {
|
||||
async upsert(scope: TenantScope, records: readonly VectorRecord[]) {
|
||||
for (const record of records) rows.set(keyOf(scope.orgId, record.id), { scope: scope.orgId, record })
|
||||
},
|
||||
async search(scope: TenantScope, query: VectorQuery): Promise<readonly VectorMatch[]> {
|
||||
const out: VectorMatch[] = []
|
||||
for (const row of rows.values()) {
|
||||
if (row.scope !== scope.orgId) continue
|
||||
const metadata = row.record.metadata ?? {}
|
||||
const filter = query.filter ?? {}
|
||||
if (!Object.entries(filter).every(([key, value]) => metadata[key] === value)) continue
|
||||
out.push({
|
||||
id: row.record.id,
|
||||
score: cosine(query.vector, row.record.vector),
|
||||
subject: row.record.subject,
|
||||
metadata,
|
||||
})
|
||||
}
|
||||
return out.sort((a, b) => b.score - a.score).slice(0, query.limit ?? 10)
|
||||
},
|
||||
async deleteBySubject(scope: TenantScope, subject: VectorSubject) {
|
||||
for (const [key, row] of rows.entries()) {
|
||||
if (
|
||||
row.scope === scope.orgId &&
|
||||
row.record.subject.type === subject.type &&
|
||||
row.record.subject.id === subject.id
|
||||
) {
|
||||
rows.delete(key)
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const dimensions = 4
|
||||
|
||||
describe("the vector store the knowledge module relies on", () => {
|
||||
it("passes the kit's two-tenant vector contract", async () => {
|
||||
const { store } = fakeVectorStore()
|
||||
const result = await runVectorStoreContract(async () => ({ store, dimensions }))
|
||||
expect(result.failures.map((failure) => `${failure.check}: ${failure.error}`)).toEqual([])
|
||||
expect(result.passed.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
// The collection-filter convention this module writes into every chunk's metadata: a search names
|
||||
// the collection and the wall holds. Pinning it here keeps the adapters' payload indexes useful.
|
||||
describe("the module's metadata conventions", () => {
|
||||
it("stores collection, filename, chunk and snippet so a cited result can be rendered", () => {
|
||||
const record: VectorRecord = {
|
||||
id: newId(),
|
||||
vector: [1, 0, 0, 0],
|
||||
subject: { type: "knowledge_document", id: newId() },
|
||||
metadata: { collection: "contracts", filename: "a.txt", chunk: 2, snippet: "passage" },
|
||||
}
|
||||
expect(Object.keys(record.metadata ?? {}).sort()).toEqual(["chunk", "collection", "filename", "snippet"])
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user