130 lines
3.9 KiB
TypeScript
130 lines
3.9 KiB
TypeScript
import { spawnSync } from "node:child_process"
|
|
import { config as loadEnv } from "dotenv"
|
|
import { ConvexHttpClient } from "convex/browser"
|
|
import { makeFunctionReference } from "convex/server"
|
|
import { scanManuals } from "../lib/manuals"
|
|
import { buildManualAssetUrl, buildThumbnailAssetUrl, getManualsAssetSource } from "../lib/manuals-storage"
|
|
|
|
loadEnv({ path: ".env.local" })
|
|
loadEnv({ path: ".env.staging", override: false })
|
|
|
|
type ManualUpsertInput = {
|
|
filename: string
|
|
path: string
|
|
manufacturer: string
|
|
category: string
|
|
size?: number
|
|
lastModified?: number
|
|
searchTerms?: string[]
|
|
commonNames?: string[]
|
|
thumbnailUrl?: string
|
|
manualUrl?: string
|
|
hasParts?: boolean
|
|
}
|
|
|
|
const UPSERT_MANUALS = makeFunctionReference<"mutation">("manuals:upsertMany")
|
|
|
|
function parseArgs(argv: string[]) {
|
|
const limitFlagIndex = argv.indexOf("--limit")
|
|
const limit = limitFlagIndex >= 0 ? Number.parseInt(argv[limitFlagIndex + 1] || "", 10) : undefined
|
|
|
|
return {
|
|
dryRun: argv.includes("--dry-run"),
|
|
skipUpload: argv.includes("--skip-upload"),
|
|
limit: Number.isFinite(limit) && limit && limit > 0 ? limit : undefined,
|
|
}
|
|
}
|
|
|
|
function readConvexUrl() {
|
|
const convexUrl = process.env.NEXT_PUBLIC_CONVEX_URL || process.env.CONVEX_URL
|
|
if (!convexUrl) {
|
|
throw new Error("NEXT_PUBLIC_CONVEX_URL is required to sync manuals into Convex.")
|
|
}
|
|
|
|
return convexUrl
|
|
}
|
|
|
|
function shouldUploadAssets() {
|
|
return Boolean(process.env.CLOUDFLARE_R2_ACCESS_KEY_ID && process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY)
|
|
}
|
|
|
|
function runAssetUpload() {
|
|
const result = spawnSync("node", ["scripts/upload-to-r2.js", "--type", "all", "--incremental"], {
|
|
cwd: process.cwd(),
|
|
stdio: "inherit",
|
|
})
|
|
|
|
if ((result.status ?? 1) !== 0) {
|
|
throw new Error("R2 upload step failed.")
|
|
}
|
|
}
|
|
|
|
function normalizeManualForConvex(manual: Awaited<ReturnType<typeof scanManuals>>[number]) {
|
|
const manualUrl = buildManualAssetUrl(manual.path)
|
|
const thumbnailUrl = manual.thumbnailUrl ? buildThumbnailAssetUrl(manual.thumbnailUrl) : undefined
|
|
|
|
return {
|
|
filename: manual.filename,
|
|
path: manual.path,
|
|
manufacturer: manual.manufacturer,
|
|
category: manual.category,
|
|
size: manual.size,
|
|
lastModified: manual.lastModified ? manual.lastModified.getTime() : undefined,
|
|
searchTerms: manual.searchTerms,
|
|
commonNames: manual.commonNames,
|
|
thumbnailUrl,
|
|
manualUrl,
|
|
hasParts: Boolean(manual.searchTerms?.some((term) => term.toLowerCase().includes("parts"))),
|
|
} satisfies ManualUpsertInput
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv.slice(2))
|
|
|
|
if (!args.skipUpload && shouldUploadAssets()) {
|
|
console.log("[manuals-sync] uploading new manuals and thumbnails to object storage")
|
|
runAssetUpload()
|
|
} else if (!args.skipUpload) {
|
|
console.log("[manuals-sync] skipping asset upload because Cloudflare R2 credentials are not configured")
|
|
}
|
|
|
|
const manuals = await scanManuals()
|
|
const selectedManuals = args.limit ? manuals.slice(0, args.limit) : manuals
|
|
const importBatch = new Date().toISOString()
|
|
const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual))
|
|
|
|
console.log("[manuals-sync] scanned manuals", {
|
|
total: manuals.length,
|
|
selected: selectedManuals.length,
|
|
assetSource: getManualsAssetSource(),
|
|
})
|
|
|
|
if (args.dryRun) {
|
|
console.log("[manuals-sync] dry run complete")
|
|
return
|
|
}
|
|
|
|
const convex = new ConvexHttpClient(readConvexUrl())
|
|
const batchSize = 100
|
|
let synced = 0
|
|
|
|
for (let index = 0; index < payload.length; index += batchSize) {
|
|
const slice = payload.slice(index, index + batchSize)
|
|
await convex.mutation(UPSERT_MANUALS, { manuals: slice })
|
|
synced += slice.length
|
|
console.log("[manuals-sync] upserted batch", {
|
|
synced,
|
|
total: payload.length,
|
|
})
|
|
}
|
|
|
|
console.log("[manuals-sync] completed", {
|
|
synced,
|
|
importBatch,
|
|
})
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error("[manuals-sync] failed", error)
|
|
process.exit(1)
|
|
})
|