import { spawnSync } from "node:child_process" import { config as loadEnv } from "dotenv" import { ConvexHttpClient } from "convex/browser" import { makeFunctionReference } from "convex/server" import { scanManuals } from "../lib/manuals" import { buildManualAssetUrl, buildThumbnailAssetUrl, getManualsAssetSource } from "../lib/manuals-storage" loadEnv({ path: ".env.local" }) loadEnv({ path: ".env.staging", override: false }) type ManualUpsertInput = { filename: string path: string manufacturer: string category: string size?: number lastModified?: number searchTerms?: string[] commonNames?: string[] thumbnailUrl?: string manualUrl?: string hasParts?: boolean } const UPSERT_MANUALS = makeFunctionReference<"mutation">("manuals:upsertMany") function parseArgs(argv: string[]) { const limitFlagIndex = argv.indexOf("--limit") const limit = limitFlagIndex >= 0 ? Number.parseInt(argv[limitFlagIndex + 1] || "", 10) : undefined return { dryRun: argv.includes("--dry-run"), skipUpload: argv.includes("--skip-upload"), limit: Number.isFinite(limit) && limit && limit > 0 ? limit : undefined, } } function readConvexUrl() { const convexUrl = process.env.NEXT_PUBLIC_CONVEX_URL || process.env.CONVEX_URL if (!convexUrl) { throw new Error("NEXT_PUBLIC_CONVEX_URL is required to sync manuals into Convex.") } return convexUrl } function shouldUploadAssets() { return Boolean(process.env.CLOUDFLARE_R2_ACCESS_KEY_ID && process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY) } function runAssetUpload() { const result = spawnSync("node", ["scripts/upload-to-r2.js", "--type", "all", "--incremental"], { cwd: process.cwd(), stdio: "inherit", }) if ((result.status ?? 1) !== 0) { throw new Error("R2 upload step failed.") } } function normalizeManualForConvex(manual: Awaited>[number]) { const manualUrl = buildManualAssetUrl(manual.path) const thumbnailUrl = manual.thumbnailUrl ? buildThumbnailAssetUrl(manual.thumbnailUrl) : undefined return { filename: manual.filename, path: manual.path, manufacturer: manual.manufacturer, category: manual.category, size: manual.size, lastModified: manual.lastModified ? manual.lastModified.getTime() : undefined, searchTerms: manual.searchTerms, commonNames: manual.commonNames, thumbnailUrl, manualUrl, hasParts: Boolean(manual.searchTerms?.some((term) => term.toLowerCase().includes("parts"))), } satisfies ManualUpsertInput } async function main() { const args = parseArgs(process.argv.slice(2)) if (!args.skipUpload && shouldUploadAssets()) { console.log("[manuals-sync] uploading new manuals and thumbnails to object storage") runAssetUpload() } else if (!args.skipUpload) { console.log("[manuals-sync] skipping asset upload because Cloudflare R2 credentials are not configured") } const manuals = await scanManuals() const selectedManuals = args.limit ? manuals.slice(0, args.limit) : manuals const importBatch = new Date().toISOString() const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual)) console.log("[manuals-sync] scanned manuals", { total: manuals.length, selected: selectedManuals.length, assetSource: getManualsAssetSource(), }) if (args.dryRun) { console.log("[manuals-sync] dry run complete") return } const convex = new ConvexHttpClient(readConvexUrl()) const batchSize = 100 let synced = 0 for (let index = 0; index < payload.length; index += batchSize) { const slice = payload.slice(index, index + batchSize) await convex.mutation(UPSERT_MANUALS, { manuals: slice }) synced += slice.length console.log("[manuals-sync] upserted batch", { synced, total: payload.length, }) } console.log("[manuals-sync] completed", { synced, importBatch, }) } main().catch((error) => { console.error("[manuals-sync] failed", error) process.exit(1) })