This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
// The enablement platform over the platform's own APIs, server-side. Eligibility reads the
|
||||
// rollout flags (agent_document_tools, agent_knowledge_tools) from the flag snapshot and probes
|
||||
// the AI tools registry for the document tool family — discovery is the cheap plan/binding/flag
|
||||
// filter, and "a working OCR endpoint does not by itself make OCR a discoverable tool". The
|
||||
// managed Qdrant link goes through the providers API (the workspace services surface): an existing
|
||||
// link is reused, so re-running the enablement flow never provisions a second service. The quota
|
||||
// pre-check has no platform API yet (the metering slice owns it); the indexing calls themselves
|
||||
// carry the typed quota_exhausted refusal at run time.
|
||||
|
||||
import { createRegistryClient, ToolIndexingError } from "@conusai-sdk/ai"
|
||||
import { ConusAIClient, type CreateServiceRequest, type ServiceInfo } from "@conusai-sdk/node-sdk"
|
||||
import type { EnablementReason } from "../contract/index.js"
|
||||
import type { DocumentAiPlatform } from "../core/ports.js"
|
||||
|
||||
export const AGENT_DOCUMENT_TOOLS_FLAG = "agent_document_tools"
|
||||
export const AGENT_KNOWLEDGE_TOOLS_FLAG = "agent_knowledge_tools"
|
||||
|
||||
const FLAG_KEYS = [AGENT_DOCUMENT_TOOLS_FLAG, AGENT_KNOWLEDGE_TOOLS_FLAG] as const
|
||||
|
||||
export interface SdkDocumentAiPlatformOptions {
|
||||
/** The deployment contract's API URL and token. */
|
||||
apiUrl: string
|
||||
token: string
|
||||
/** The organization's slug and the workspace the app links services into. */
|
||||
orgSlug: string
|
||||
workspaceSlug: string
|
||||
/** The deployment environment, for the operator-configured Qdrant variables. */
|
||||
env?: Record<string, string | undefined>
|
||||
fetch?: typeof fetch
|
||||
}
|
||||
|
||||
export function sdkDocumentAiPlatform(options: SdkDocumentAiPlatformOptions): DocumentAiPlatform {
|
||||
const http = options.fetch ?? fetch
|
||||
|
||||
const rolloutFlagsOff = async (): Promise<string | null> => {
|
||||
try {
|
||||
const response = await http(new URL(`${options.apiUrl.replace(/\/+$/, "")}/flags/snapshot`), {
|
||||
headers: { authorization: `Bearer ${options.token}`, accept: "application/json" },
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
})
|
||||
if (!response.ok) return null // the flags service is not the eligibility authority
|
||||
const body = (await response.json()) as { flags?: Array<{ key?: string; enabled?: boolean }> }
|
||||
for (const flag of body.flags ?? []) {
|
||||
if (
|
||||
typeof flag.key === "string" &&
|
||||
(FLAG_KEYS as readonly string[]).includes(flag.key) &&
|
||||
flag.enabled === false
|
||||
) {
|
||||
return flag.key
|
||||
}
|
||||
}
|
||||
return null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async checkEligibility(): Promise<{ ok: true } | { ok: false; reason: EnablementReason; detail?: string }> {
|
||||
// The rollout flags first: a kill switch or a disabled feature refuses before any probe.
|
||||
const off = await rolloutFlagsOff()
|
||||
if (off !== null) return { ok: false, reason: "feature_disabled", detail: `the ${off} flag is off` }
|
||||
// The plan probe: can this deployment discover the document tool family at all?
|
||||
try {
|
||||
const registry = createRegistryClient({
|
||||
env: { CONUSAI_API_URL: options.apiUrl, CONUSAI_API_TOKEN: options.token },
|
||||
fetch: http,
|
||||
})
|
||||
const found = await registry.tools.search("read and index documents")
|
||||
if (found.length === 0) {
|
||||
return { ok: false, reason: "plan_required", detail: "the ai_tools plan capability lists no document tools" }
|
||||
}
|
||||
return { ok: true }
|
||||
} catch (error) {
|
||||
if (error instanceof ToolIndexingError) {
|
||||
return { ok: false, reason: "feature_disabled", detail: "the tool registry is still indexing" }
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: "plan_required",
|
||||
detail: "the AI tools registry did not answer the eligibility probe",
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
async indexingQuota() {
|
||||
// No pre-check API exists on the platform yet; the call-time quota refusal is the hard
|
||||
// gate. Until the limit endpoint lands, the admission check answers "within quota".
|
||||
return Number.MAX_SAFE_INTEGER
|
||||
},
|
||||
|
||||
async linkManagedQdrant() {
|
||||
const client = new ConusAIClient({ baseUrl: options.apiUrl, apiKey: options.token })
|
||||
const path = { org_slug: options.orgSlug, workspace_slug: options.workspaceSlug }
|
||||
const existing = await client.listServices({ path })
|
||||
const services = (existing.data ?? []) as ServiceInfo[]
|
||||
const reused = services.find((service) => service.service_type === "qdrant" && service.status !== "deleted")
|
||||
if (reused) return { serviceId: String(reused.id), url: null }
|
||||
const body: CreateServiceRequest = { name: "knowledge-vectors", service_type: "qdrant", parameters: {} }
|
||||
const created = await client.createService({ path, body })
|
||||
const status = created.response?.status ?? 0
|
||||
if (status === 403) {
|
||||
throw Object.assign(new Error("the plan does not allow a managed Qdrant service"), {
|
||||
reason: "plan_required" as const,
|
||||
})
|
||||
}
|
||||
if (status !== 201 || !created.data) {
|
||||
throw new Error(`the providers API answered ${status} for the Qdrant link`)
|
||||
}
|
||||
return { serviceId: String(created.data.id), url: null }
|
||||
},
|
||||
|
||||
async probeQdrant(url, apiKey) {
|
||||
try {
|
||||
const response = await http(new URL("/collections", url), {
|
||||
headers: { "api-key": apiKey },
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
})
|
||||
if (response.ok) return { ok: true }
|
||||
return { ok: false, detail: `the endpoint answered ${response.status}` }
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
detail:
|
||||
error instanceof Error ? (error.message.split("\n")[0] ?? error.message) : "the endpoint did not answer",
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
operatorQdrant() {
|
||||
// The operator-configured endpoint: exactly the variables a linked Qdrant service injects,
|
||||
// set by hand in the deployment instead. The key is validated by probing; it is never
|
||||
// persisted — only the endpoint is recorded.
|
||||
const env = options.env ?? process.env
|
||||
const url = env.QDRANT_URL?.trim()
|
||||
const apiKey = env.QDRANT_API_KEY?.trim()
|
||||
if (!url || !apiKey) return null
|
||||
return { url, apiKey }
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// The knowledge jobs and the subscribers that enqueue them.
|
||||
//
|
||||
// A web request writes the document row and publishes the outbox event in one transaction; the
|
||||
// module's own subscriber (running in the worker's outbox relay) turns the event into the job,
|
||||
// with the (collection, file) pair as the singleton key — a redelivered event adds no second job.
|
||||
// The tenant comes from the container, never from the payload (decision AT-F6).
|
||||
|
||||
import { type EventContext, internal, type JobDef, subscribe } from "@conusai-kit/core"
|
||||
import { JOB_QUEUE } from "@conusai-kit/data"
|
||||
import { z } from "zod"
|
||||
import { documentIndexRequested, documentRemoveRequested, INDEX_JOB, REMOVE_JOB } from "../contract/index.js"
|
||||
import { indexDocument, removeDocument } from "../core/index-job.js"
|
||||
import { KNOWLEDGE_SCOPE, type KnowledgeScope } from "../core/ports.js"
|
||||
import { knowledgeSearchDeps } from "./procedures.js"
|
||||
|
||||
export const documentPayload = z.object({
|
||||
documentId: z.string(),
|
||||
collectionName: z.string(),
|
||||
fileId: z.string(),
|
||||
})
|
||||
export type DocumentJobPayload = z.output<typeof documentPayload>
|
||||
|
||||
export const indexJobKey = (collectionName: string, fileId: string): string =>
|
||||
`knowledge.index.${collectionName}.${fileId}`
|
||||
export const removeJobKey = (collectionName: string, fileId: string): string =>
|
||||
`knowledge.remove.${collectionName}.${fileId}`
|
||||
|
||||
function projectScopeOf(ctx: EventContext, payload: { workspaceId: string; projectId: string }): KnowledgeScope {
|
||||
const ambient = ctx.container.resolve(KNOWLEDGE_SCOPE)
|
||||
if (ambient.level !== "project") throw internal("a knowledge event resolves to a project scope")
|
||||
return { ...ambient, workspaceId: payload.workspaceId, projectId: payload.projectId }
|
||||
}
|
||||
|
||||
export const indexJob: JobDef<DocumentJobPayload> = {
|
||||
payload: documentPayload,
|
||||
async handler(payload, ctx) {
|
||||
const scope = ctx.container.resolve(KNOWLEDGE_SCOPE)
|
||||
if (scope.level !== "project") throw internal(`a knowledge job runs in a project scope, not a ${scope.level} one`)
|
||||
await indexDocument(knowledgeSearchDeps(ctx.container), scope, payload.documentId, {
|
||||
...(ctx.signal === undefined ? {} : { signal: ctx.signal }),
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
export const removeJob: JobDef<DocumentJobPayload> = {
|
||||
payload: documentPayload,
|
||||
async handler(payload, ctx) {
|
||||
const scope = ctx.container.resolve(KNOWLEDGE_SCOPE)
|
||||
if (scope.level !== "project") throw internal(`a knowledge job runs in a project scope, not a ${scope.level} one`)
|
||||
await removeDocument(knowledgeSearchDeps(ctx.container), scope, payload.documentId)
|
||||
},
|
||||
}
|
||||
|
||||
async function onDocumentIndexRequested(
|
||||
event: {
|
||||
name: string
|
||||
payload: { workspaceId: string; projectId: string; documentId: string; collectionName: string; fileId: string }
|
||||
},
|
||||
ctx: EventContext,
|
||||
): Promise<void> {
|
||||
const scope = projectScopeOf(ctx, event.payload)
|
||||
await ctx.container.resolve(JOB_QUEUE).enqueue(
|
||||
scope,
|
||||
INDEX_JOB,
|
||||
{
|
||||
documentId: event.payload.documentId,
|
||||
collectionName: event.payload.collectionName,
|
||||
fileId: event.payload.fileId,
|
||||
},
|
||||
{
|
||||
key: indexJobKey(event.payload.collectionName, event.payload.fileId),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async function onDocumentRemoveRequested(
|
||||
event: {
|
||||
name: string
|
||||
payload: { workspaceId: string; projectId: string; documentId: string; collectionName: string; fileId: string }
|
||||
},
|
||||
ctx: EventContext,
|
||||
): Promise<void> {
|
||||
const scope = projectScopeOf(ctx, event.payload)
|
||||
await ctx.container.resolve(JOB_QUEUE).enqueue(
|
||||
scope,
|
||||
REMOVE_JOB,
|
||||
{
|
||||
documentId: event.payload.documentId,
|
||||
collectionName: event.payload.collectionName,
|
||||
fileId: event.payload.fileId,
|
||||
},
|
||||
{
|
||||
key: removeJobKey(event.payload.collectionName, event.payload.fileId),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
export const KNOWLEDGE_JOBS = { [INDEX_JOB]: indexJob, [REMOVE_JOB]: removeJob }
|
||||
export const KNOWLEDGE_SUBSCRIPTIONS = [
|
||||
subscribe(documentIndexRequested, onDocumentIndexRequested),
|
||||
subscribe(documentRemoveRequested, onDocumentRemoveRequested),
|
||||
]
|
||||
@@ -0,0 +1,116 @@
|
||||
// The managed search service: the platform's documents-index service through the kit-platform
|
||||
// Knowledge client. Indexing and search run server-side there (the service owns chunking,
|
||||
// embeddings and the vector store); this module supplies the job lifecycle and the citations it
|
||||
// answers with.
|
||||
//
|
||||
// Contract notes (the service's real shapes): the index submission carries the content's SHA-256
|
||||
// (computed here from the job's bytes) as its version assertion; a quota denial surfaces on the
|
||||
// job, which the poll below reads as the job's typed error. Blob pathnames and returned citations
|
||||
// resolve through the files module's scoped rows, so an opaque file id never becomes a pathname
|
||||
// and a foreign or stale service reference never becomes a valid app hit.
|
||||
|
||||
import { createHash } from "node:crypto"
|
||||
import {
|
||||
type FilesDeps,
|
||||
findFileStorageByKey,
|
||||
getFileStorage,
|
||||
platformBlobPathname,
|
||||
platformBlobStorageKey,
|
||||
} from "@conusai-kit/files"
|
||||
import { type Knowledge, KnowledgeError } from "@conusai-kit/platform"
|
||||
import type { KnowledgeHit, KnowledgeScope, KnowledgeSearchService } from "../core/ports.js"
|
||||
|
||||
export function platformKnowledgeSearch(client: Knowledge, files: FilesDeps): KnowledgeSearchService {
|
||||
const unwrap = (error: unknown): never => {
|
||||
// The client's errors are already typed by reason; a non-knowledge failure is unavailable.
|
||||
if (error instanceof KnowledgeError) throw error
|
||||
throw new KnowledgeError(
|
||||
"runtime_unavailable",
|
||||
error instanceof Error ? error.message : "the knowledge service failed",
|
||||
true,
|
||||
)
|
||||
}
|
||||
const documentRef = async (scope: KnowledgeScope, fileId: string): Promise<string> => {
|
||||
const reference = await getFileStorage(files, scope, fileId)
|
||||
return platformBlobPathname(scope, reference.storageKey)
|
||||
}
|
||||
|
||||
return {
|
||||
async readiness() {
|
||||
try {
|
||||
const answer = await client.readiness()
|
||||
return { state: answer.state, reasons: [...answer.reasons] }
|
||||
} catch (error) {
|
||||
return {
|
||||
state: "unavailable",
|
||||
reasons: [error instanceof Error ? (error.message.split("\n")[0] ?? error.message) : "not ready"],
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
async index(scope, input) {
|
||||
try {
|
||||
const { jobId } = await client.index({
|
||||
collection: input.collectionName,
|
||||
documentRef: await documentRef(scope, input.fileId),
|
||||
options: {
|
||||
visibility: "public",
|
||||
contentSha256: createHash("sha256").update(input.bytes).digest("hex"),
|
||||
},
|
||||
})
|
||||
// The service's job is durable; poll until it leaves the queue. A caller-side cancel is
|
||||
// the job's own AbortSignal, checked between polls.
|
||||
for (;;) {
|
||||
const job = await client.job(jobId)
|
||||
if (job.status === "succeeded") return
|
||||
if (job.status === "failed")
|
||||
throw new KnowledgeError(
|
||||
job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable",
|
||||
job.error ?? "the indexing job failed",
|
||||
)
|
||||
if (job.status === "cancelled")
|
||||
throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled")
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000))
|
||||
}
|
||||
} catch (error) {
|
||||
return unwrap(error)
|
||||
}
|
||||
},
|
||||
|
||||
async search(scope, input) {
|
||||
try {
|
||||
const results = await client.search({
|
||||
collection: input.collectionName,
|
||||
query: input.query,
|
||||
limit: input.limit,
|
||||
})
|
||||
const hits: KnowledgeHit[] = []
|
||||
for (const [at, hit] of results.entries()) {
|
||||
const storageKey = platformBlobStorageKey(scope, hit.documentRef)
|
||||
if (storageKey === null) continue
|
||||
const reference = await findFileStorageByKey(files, scope, storageKey)
|
||||
if (reference === null) continue
|
||||
hits.push({
|
||||
fileId: reference.file.fileId,
|
||||
filename: reference.file.filename,
|
||||
collectionName: input.collectionName,
|
||||
score: hit.score,
|
||||
chunk: at + 1,
|
||||
snippet: hit.snippet,
|
||||
})
|
||||
}
|
||||
return hits
|
||||
} catch (error) {
|
||||
return unwrap(error)
|
||||
}
|
||||
},
|
||||
|
||||
async remove(scope, input) {
|
||||
try {
|
||||
await client.remove(await documentRef(scope, input.fileId))
|
||||
} catch (error) {
|
||||
return unwrap(error)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// The handlers of the knowledge module's procedures: each resolves its ports from the request's
|
||||
// container and calls one use case. No rule lives here.
|
||||
|
||||
import type { Container } from "@conusai-kit/core"
|
||||
import { implement } from "@conusai-kit/core"
|
||||
import { DATA_SOURCE } from "@conusai-kit/data"
|
||||
import { filesDeps } from "@conusai-kit/files"
|
||||
import {
|
||||
collectionCreate,
|
||||
collectionList,
|
||||
documentIndex,
|
||||
documentList,
|
||||
documentRemove,
|
||||
knowledgeSearch,
|
||||
knowledgeStatus,
|
||||
settingsDisableDocumentAi,
|
||||
settingsEnableDocumentAi,
|
||||
settingsGet,
|
||||
} from "../contract/index.js"
|
||||
import { createCollection, listCollections } from "../core/collections.js"
|
||||
import { listDocuments, requestIndex, requestRemove } from "../core/documents.js"
|
||||
import { flagsOf, KNOWLEDGE_ENABLEMENT, KNOWLEDGE_SEARCH, type KnowledgeDeps } from "../core/ports.js"
|
||||
import { searchKnowledge } from "../core/search.js"
|
||||
import { disableDocumentAi, enableDocumentAi, getSettings, type SettingsDeps } from "../core/settings.js"
|
||||
import { knowledgeStatusOf } from "../core/status.js"
|
||||
|
||||
/** The ports a write or read handler needs. Never the search service: that port is an effect, and
|
||||
* resolving it from a write is a kind violation. */
|
||||
export function knowledgeDeps(container: Container): KnowledgeDeps {
|
||||
return {
|
||||
data: container.resolve(DATA_SOURCE),
|
||||
files: filesDeps(container),
|
||||
flags: flagsOf(container),
|
||||
}
|
||||
}
|
||||
|
||||
/** The effect handler's deps: the search service, bound in the processes that may leave the process. */
|
||||
export function knowledgeSearchDeps(container: Container): KnowledgeDeps {
|
||||
return { ...knowledgeDeps(container), search: container.resolve(KNOWLEDGE_SEARCH) }
|
||||
}
|
||||
|
||||
function settingsDeps(container: Container): SettingsDeps {
|
||||
return { data: container.resolve(DATA_SOURCE), platform: container.resolve(KNOWLEDGE_ENABLEMENT) }
|
||||
}
|
||||
|
||||
export const collectionListImpl = implement(collectionList, ({ ctx }) =>
|
||||
listCollections(knowledgeDeps(ctx.container), ctx.scope),
|
||||
)
|
||||
|
||||
export const collectionCreateImpl = implement(collectionCreate, ({ input, ctx }) =>
|
||||
createCollection(knowledgeDeps(ctx.container), ctx.scope, input.name),
|
||||
)
|
||||
|
||||
export const documentIndexImpl = implement(documentIndex, ({ input, ctx }) =>
|
||||
requestIndex(knowledgeDeps(ctx.container), ctx.scope, input),
|
||||
)
|
||||
|
||||
export const documentListImpl = implement(documentList, ({ input, ctx }) =>
|
||||
listDocuments(knowledgeDeps(ctx.container), ctx.scope, input.collectionName),
|
||||
)
|
||||
|
||||
export const documentRemoveImpl = implement(documentRemove, ({ input, ctx }) =>
|
||||
requestRemove(knowledgeDeps(ctx.container), ctx.scope, input),
|
||||
)
|
||||
|
||||
export const knowledgeSearchImpl = implement(knowledgeSearch, ({ input, ctx }) =>
|
||||
searchKnowledge(knowledgeSearchDeps(ctx.container), ctx.scope, {
|
||||
collectionName: input.collectionName,
|
||||
query: input.query,
|
||||
...(input.limit === undefined ? {} : { limit: input.limit }),
|
||||
}),
|
||||
)
|
||||
|
||||
export const knowledgeStatusImpl = implement(knowledgeStatus, ({ ctx }) =>
|
||||
knowledgeStatusOf(knowledgeSearchDeps(ctx.container), ctx.scope),
|
||||
)
|
||||
|
||||
export const settingsGetImpl = implement(settingsGet, ({ ctx }) => getSettings(settingsDeps(ctx.container), ctx.scope))
|
||||
|
||||
export const settingsEnableDocumentAiImpl = implement(settingsEnableDocumentAi, async ({ ctx }) => {
|
||||
const settings = await enableDocumentAi(settingsDeps(ctx.container), ctx.scope)
|
||||
return { enabled: settings.enabled, backend: settings.backend ?? "vectors" }
|
||||
})
|
||||
|
||||
export const settingsDisableDocumentAiImpl = implement(settingsDisableDocumentAi, ({ ctx }) =>
|
||||
disableDocumentAi(settingsDeps(ctx.container), ctx.scope),
|
||||
)
|
||||
|
||||
export const KNOWLEDGE_PROCEDURE_IMPLS = [
|
||||
collectionListImpl,
|
||||
collectionCreateImpl,
|
||||
documentIndexImpl,
|
||||
documentListImpl,
|
||||
documentRemoveImpl,
|
||||
knowledgeSearchImpl,
|
||||
knowledgeStatusImpl,
|
||||
settingsGetImpl,
|
||||
settingsEnableDocumentAiImpl,
|
||||
settingsDisableDocumentAiImpl,
|
||||
]
|
||||
@@ -0,0 +1,41 @@
|
||||
// The search service of a deployment whose vectors backend has no vector store (no Qdrant linked,
|
||||
// no explicit pgvector choice). It is not a fake: it is the honest answer of such a deployment.
|
||||
// Readiness says `unavailable` with the reason a person can act on; an index job fails its row with
|
||||
// the stable `vector_store_unavailable` code instead of retrying forever; a search or a removal is
|
||||
// refused with a typed problem. Nothing is written anywhere, so choosing a store later (enabling
|
||||
// document AI links a managed Qdrant) starts from a clean state.
|
||||
|
||||
import { AppError } from "@conusai-kit/core"
|
||||
import type { KnowledgeSearchService } from "../core/ports.js"
|
||||
|
||||
function unavailable(reason: string): AppError {
|
||||
return new AppError({
|
||||
type: "/problems/runtime_unavailable",
|
||||
title: "The knowledge search service is unavailable",
|
||||
status: 503,
|
||||
code: "runtime_unavailable",
|
||||
detail: reason,
|
||||
})
|
||||
}
|
||||
|
||||
export function unavailableKnowledgeSearch(reason: string): KnowledgeSearchService {
|
||||
return {
|
||||
async readiness() {
|
||||
return { state: "unavailable", reasons: [reason] }
|
||||
},
|
||||
|
||||
async index() {
|
||||
// A permanent failure: the row records why, and a retry of the same job cannot succeed until
|
||||
// the deployment binds a store (a new deployment, which re-runs the job).
|
||||
throw Object.assign(new Error(reason), { permanent: "vector_store_unavailable" })
|
||||
},
|
||||
|
||||
async search() {
|
||||
throw unavailable(reason)
|
||||
},
|
||||
|
||||
async remove() {
|
||||
throw unavailable(reason)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// The local vector search service: the knowledge module's own backend. Chunks are embedded
|
||||
// through the AI gateway's embeddings endpoint and stored in the app's vector adapter (Qdrant when
|
||||
// a Qdrant service is linked, pgvector otherwise); a query is embedded the same way and searched
|
||||
// with the collection as a metadata filter, so the tenant wall is the adapter's job, never the
|
||||
// caller's. Snippets and chunk positions ride in the metadata and are what a citation shows.
|
||||
|
||||
import { createHash } from "node:crypto"
|
||||
import { internal } from "@conusai-kit/core"
|
||||
import type { VectorStore } from "@conusai-kit/data"
|
||||
import { chunkText } from "../core/chunk.js"
|
||||
import type { KnowledgeEmbeddings, KnowledgeHit, KnowledgeScope, KnowledgeSearchService } from "../core/ports.js"
|
||||
|
||||
const SUBJECT = "knowledge_document"
|
||||
const SNIPPET = 300
|
||||
|
||||
export interface VectorKnowledgeSearchOptions {
|
||||
embeddings: KnowledgeEmbeddings
|
||||
store: VectorStore
|
||||
}
|
||||
|
||||
export function vectorKnowledgeSearch(options: VectorKnowledgeSearchOptions): KnowledgeSearchService {
|
||||
const { embeddings, store } = options
|
||||
|
||||
return {
|
||||
async readiness() {
|
||||
// The local backend is ready when it can answer a search; the embedding model is the one
|
||||
// external piece, and an embeddings call is too expensive for a health probe — so readiness
|
||||
// here is the store's own, and a misconfigured embedding model fails the first real call.
|
||||
return { state: "ready", reasons: [] }
|
||||
},
|
||||
|
||||
async index(scope, input) {
|
||||
const text = new TextDecoder("utf-8", { fatal: false }).decode(input.bytes)
|
||||
const chunks = chunkText(text)
|
||||
const vectors = await embeddings.embed(
|
||||
chunks.map((chunk) => chunk.text),
|
||||
{ inputType: "passage" },
|
||||
)
|
||||
if (vectors.length !== chunks.length) {
|
||||
throw internal(`the embeddings answer held ${vectors.length} vectors for ${chunks.length} chunks`)
|
||||
}
|
||||
const vectorAt = (at: number): readonly number[] => {
|
||||
const vector = vectors[at]
|
||||
if (vector === undefined) throw internal("the embeddings answer held no vector for a chunk")
|
||||
return vector
|
||||
}
|
||||
await store.upsert(
|
||||
scope,
|
||||
chunks.map((chunk, at) => ({
|
||||
id: stableChunkId(input.fileId, chunk.position),
|
||||
vector: vectorAt(at),
|
||||
subject: { type: SUBJECT, id: input.fileId },
|
||||
metadata: {
|
||||
collection: input.collectionName,
|
||||
filename: input.filename,
|
||||
chunk: chunk.position,
|
||||
snippet: chunk.text.slice(0, SNIPPET),
|
||||
},
|
||||
})),
|
||||
)
|
||||
},
|
||||
|
||||
async search(scope, input) {
|
||||
const vectors = await embeddings.embed([input.query], { inputType: "query" })
|
||||
const vector = vectors[0]
|
||||
if (!vector) return []
|
||||
const matches = await store.search(scope, {
|
||||
vector,
|
||||
limit: input.limit,
|
||||
filter: { collection: input.collectionName },
|
||||
})
|
||||
const hits: KnowledgeHit[] = []
|
||||
for (const match of matches) {
|
||||
const metadata = match.metadata ?? {}
|
||||
const snippet = typeof metadata.snippet === "string" ? metadata.snippet : ""
|
||||
const chunk = typeof metadata.chunk === "number" ? metadata.chunk : 1
|
||||
const filename = typeof metadata.filename === "string" ? metadata.filename : match.subject.id
|
||||
const collectionName = typeof metadata.collection === "string" ? metadata.collection : input.collectionName
|
||||
hits.push({ fileId: match.subject.id, filename, collectionName, score: match.score, chunk, snippet })
|
||||
}
|
||||
return hits
|
||||
},
|
||||
|
||||
async remove(scope, input) {
|
||||
await store.deleteBySubject(scope, { type: SUBJECT, id: input.fileId })
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// A deterministic chunk id keeps a re-indexed chunk under one identity; the vector adapters
|
||||
// namespace it per organization themselves.
|
||||
function stableChunkId(fileId: string, position: number): string {
|
||||
const hex = createHash("sha256").update(`${fileId}:${position}`).digest("hex").slice(0, 32)
|
||||
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`
|
||||
}
|
||||
|
||||
// Types referenced by the composition root when it binds the adapter.
|
||||
export type { KnowledgeHit, KnowledgeScope }
|
||||
Reference in New Issue
Block a user