121 lines
4.8 KiB
JavaScript
121 lines
4.8 KiB
JavaScript
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
// Convex exports require the admin CLI, so this runs beside (not inside) Convex functions.
|
|
|
|
import { execFile } from "node:child_process"
|
|
import { createReadStream } from "node:fs"
|
|
import { mkdtemp, rm, stat } from "node:fs/promises"
|
|
import { tmpdir } from "node:os"
|
|
import { join } from "node:path"
|
|
import { Readable } from "node:stream"
|
|
import { promisify } from "node:util"
|
|
import { createObjectStore } from "@conusai-kit/platform"
|
|
|
|
const execFileAsync = promisify(execFile)
|
|
|
|
export async function backupOnce(env = process.env, now = new Date()) {
|
|
const url = requiredFrom(env, "CONVEX_SELF_HOSTED_URL")
|
|
const adminKey = requiredFrom(env, "CONVEX_SELF_HOSTED_ADMIN_KEY")
|
|
const apiUrl = requiredFrom(env, "CONUSAI_API_URL")
|
|
const token = requiredFrom(env, "CONUSAI_API_TOKEN")
|
|
const projectId = requiredFrom(env, "CONUSAI_PROJECT_ID")
|
|
const retention = positiveInt(env.CONVEX_BACKUP_RETENTION ?? "30", "CONVEX_BACKUP_RETENTION")
|
|
const directory = await mkdtemp(join(tmpdir(), "conusai-convex-backup-"))
|
|
const archive = join(directory, "convex-export.zip")
|
|
try {
|
|
await execFileAsync(
|
|
env.CONVEX_CLI ?? "node_modules/.bin/convex",
|
|
["export", "--include-file-storage", "--path", archive],
|
|
{
|
|
cwd: env.CONVEX_APP_DIR ?? process.cwd(),
|
|
env: { ...env, CONVEX_SELF_HOSTED_URL: url, CONVEX_SELF_HOSTED_ADMIN_KEY: adminKey },
|
|
maxBuffer: 8 * 1024 * 1024,
|
|
},
|
|
)
|
|
const { size: sizeBytes } = await stat(archive)
|
|
if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 1)
|
|
throw new Error("Convex export archive is empty or too large")
|
|
const stamp = now.toISOString().replaceAll(/[:.]/g, "-")
|
|
const pathname = `convex-backups/${projectId}/${stamp}.zip`
|
|
const store = createObjectStore({
|
|
CONUSAI_API_URL: apiUrl,
|
|
CONUSAI_API_TOKEN: token,
|
|
CONUSAI_PROJECT_ID: projectId,
|
|
})
|
|
const session = await store.createUpload({ pathname, contentType: "application/zip", sizeBytes })
|
|
const parts = []
|
|
const chunkSize = session.upload.mode === "multipart" ? session.upload.partSizeBytes : sizeBytes
|
|
for (let offset = 0, part = 1; offset < sizeBytes; offset += chunkSize, part += 1) {
|
|
const partSize = Math.min(chunkSize, sizeBytes - offset)
|
|
const source = createReadStream(archive, { start: offset, end: offset + partSize - 1 })
|
|
const uploaded = await store.uploadContent(session.uploadId, Readable.toWeb(source), {
|
|
...(session.upload.mode === "multipart" ? { part } : {}),
|
|
size: partSize,
|
|
})
|
|
if (session.upload.mode === "multipart") parts.push({ partNumber: part, etag: uploaded.etag })
|
|
}
|
|
const completed = await store.completeUpload(
|
|
session.uploadId,
|
|
session.upload.mode === "multipart" ? { parts } : undefined,
|
|
)
|
|
await prune(store, `convex-backups/${projectId}/`, retention)
|
|
return { pathname, sizeBytes, sha256Digest: completed.sha256Digest }
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true })
|
|
}
|
|
}
|
|
|
|
function requiredFrom(env, name) {
|
|
const value = env[name]
|
|
if (!value) throw new Error(`${name} is required for Convex backups`)
|
|
return value
|
|
}
|
|
|
|
function positiveInt(value, name) {
|
|
const parsed = Number(value)
|
|
if (!Number.isSafeInteger(parsed) || parsed < 1) throw new Error(`${name} must be a positive integer`)
|
|
return parsed
|
|
}
|
|
|
|
async function prune(store, prefix, retention) {
|
|
const blobs = []
|
|
let cursor
|
|
do {
|
|
const page = await store.list({ prefix, limit: 1000, ...(cursor ? { cursor } : {}) })
|
|
blobs.push(...page.blobs)
|
|
cursor = page.hasMore ? page.cursor : undefined
|
|
} while (cursor)
|
|
blobs.sort((a, b) => b.pathname.localeCompare(a.pathname))
|
|
const expired = blobs.slice(retention).map((blob) => blob.url)
|
|
if (expired.length) await store.del(expired)
|
|
}
|
|
|
|
if (process.argv[1] && new URL(import.meta.url).pathname === process.argv[1]) {
|
|
if (process.argv.includes("--once")) {
|
|
try {
|
|
const result = await backupOnce()
|
|
console.info(
|
|
`Convex backup complete: ${result.pathname} (${result.sizeBytes} bytes, sha256 ${result.sha256Digest})`,
|
|
)
|
|
process.exit(0)
|
|
} catch {
|
|
console.error("Convex backup failed (CLI, Blob, or configuration error)")
|
|
process.exit(1)
|
|
}
|
|
}
|
|
const interval =
|
|
positiveInt(process.env.CONVEX_BACKUP_INTERVAL_HOURS ?? "24", "CONVEX_BACKUP_INTERVAL_HOURS") * 60 * 60 * 1000
|
|
const run = async () => {
|
|
try {
|
|
const result = await backupOnce()
|
|
console.info(
|
|
`Convex backup complete: ${result.pathname} (${result.sizeBytes} bytes, sha256 ${result.sha256Digest})`,
|
|
)
|
|
} catch {
|
|
// CLI errors can include their argv, which contains the admin key. Never log raw errors.
|
|
console.error("Convex backup failed (CLI, Blob, or configuration error)")
|
|
}
|
|
setTimeout(() => void run(), interval)
|
|
}
|
|
void run()
|
|
}
|