ConusAI vendor v0.1.0-dev.20261027+9b1131fb

modules/knowledge/adapters/vectors.ts

@@ -0,0 +1,98 @@
+// The local vector search service: the knowledge module's own backend. Chunks are embedded
+// through the AI gateway's embeddings endpoint and stored in the app's vector adapter (Qdrant when
+// a Qdrant service is linked, pgvector otherwise); a query is embedded the same way and searched
+// with the collection as a metadata filter, so the tenant wall is the adapter's job, never the
+// caller's. Snippets and chunk positions ride in the metadata and are what a citation shows.
+
+import { createHash } from "node:crypto"
+import { internal } from "@conusai-kit/core"
+import type { VectorStore } from "@conusai-kit/data"
+import { chunkText } from "../core/chunk.js"
+import type { KnowledgeEmbeddings, KnowledgeHit, KnowledgeScope, KnowledgeSearchService } from "../core/ports.js"
+
+const SUBJECT = "knowledge_document"
+const SNIPPET = 300
+
+export interface VectorKnowledgeSearchOptions {
+  embeddings: KnowledgeEmbeddings
+  store: VectorStore
+}
+
+export function vectorKnowledgeSearch(options: VectorKnowledgeSearchOptions): KnowledgeSearchService {
+  const { embeddings, store } = options
+
+  return {
+    async readiness() {
+      // The local backend is ready when it can answer a search; the embedding model is the one
+      // external piece, and an embeddings call is too expensive for a health probe — so readiness
+      // here is the store's own, and a misconfigured embedding model fails the first real call.
+      return { state: "ready", reasons: [] }
+    },
+
+    async index(scope, input) {
+      const text = new TextDecoder("utf-8", { fatal: false }).decode(input.bytes)
+      const chunks = chunkText(text)
+      const vectors = await embeddings.embed(
+        chunks.map((chunk) => chunk.text),
+        { inputType: "passage" },
+      )
+      if (vectors.length !== chunks.length) {
+        throw internal(`the embeddings answer held ${vectors.length} vectors for ${chunks.length} chunks`)
+      }
+      const vectorAt = (at: number): readonly number[] => {
+        const vector = vectors[at]
+        if (vector === undefined) throw internal("the embeddings answer held no vector for a chunk")
+        return vector
+      }
+      await store.upsert(
+        scope,
+        chunks.map((chunk, at) => ({
+          id: stableChunkId(input.fileId, chunk.position),
+          vector: vectorAt(at),
+          subject: { type: SUBJECT, id: input.fileId },
+          metadata: {
+            collection: input.collectionName,
+            filename: input.filename,
+            chunk: chunk.position,
+            snippet: chunk.text.slice(0, SNIPPET),
+          },
+        })),
+      )
+    },
+
+    async search(scope, input) {
+      const vectors = await embeddings.embed([input.query], { inputType: "query" })
+      const vector = vectors[0]
+      if (!vector) return []
+      const matches = await store.search(scope, {
+        vector,
+        limit: input.limit,
+        filter: { collection: input.collectionName },
+      })
+      const hits: KnowledgeHit[] = []
+      for (const match of matches) {
+        const metadata = match.metadata ?? {}
+        const snippet = typeof metadata.snippet === "string" ? metadata.snippet : ""
+        const chunk = typeof metadata.chunk === "number" ? metadata.chunk : 1
+        const filename = typeof metadata.filename === "string" ? metadata.filename : match.subject.id
+        const collectionName = typeof metadata.collection === "string" ? metadata.collection : input.collectionName
+        hits.push({ fileId: match.subject.id, filename, collectionName, score: match.score, chunk, snippet })
+      }
+      return hits
+    },
+
+    async remove(scope, input) {
+      await store.deleteBySubject(scope, { type: SUBJECT, id: input.fileId })
+    },
+  }
+}
+
+// A deterministic chunk id keeps a re-indexed chunk under one identity; the vector adapters
+// namespace it per organization themselves.
+function stableChunkId(fileId: string, position: number): string {
+  const hex = createHash("sha256").update(`${fileId}:${position}`).digest("hex").slice(0, 32)
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`
+}
+
+// Types referenced by the composition root when it binds the adapter.
+export type { KnowledgeHit, KnowledgeScope }