import { readFile } from "node:fs/promises" import { join } from "node:path" import { config as loadEnv } from "dotenv" import { ConvexHttpClient } from "convex/browser" import { makeFunctionReference } from "convex/server" import { scanManuals } from "../lib/manuals" import { getManualsAssetSource, buildManualAssetUrl, buildThumbnailAssetUrl, } from "../lib/manuals-storage" import { businessConfig } from "../lib/seo-config" import { getSiteDomain } from "../lib/site-config" import { selectManualsForSite } from "../lib/manuals-site-selection" import { canonicalizeTenantDomain, resolveManualsTenantDomain, tenantDomainVariants, } from "../lib/manuals-tenant" import { getManualsFilesRoot, getManualsThumbnailsRoot, } from "../lib/manuals-paths" import { hasManualObjectStorageConfig, manualAssetExists, uploadManualAsset, } from "../lib/manuals-object-storage" loadEnv({ path: ".env.local" }) loadEnv({ path: ".env.staging", override: false }) type ManualUpsertInput = { filename: string path: string manufacturer: string category: string size?: number lastModified?: number searchTerms?: string[] commonNames?: string[] thumbnailUrl?: string manualUrl?: string hasParts?: boolean assetSource?: string sourcePath?: string sourceSite?: string sourceDomain?: string siteVisibility?: string[] importBatch?: string } const UPSERT_MANUALS = makeFunctionReference<"mutation">("manuals:upsertMany") function parseArgs(argv: string[]) { const limitFlagIndex = argv.indexOf("--limit") const limit = limitFlagIndex >= 0 ? Number.parseInt(argv[limitFlagIndex + 1] || "", 10) : undefined return { dryRun: argv.includes("--dry-run"), skipUpload: argv.includes("--skip-upload"), limit: Number.isFinite(limit) && limit && limit > 0 ? limit : undefined, } } function readConvexUrl() { const convexUrl = process.env.NEXT_PUBLIC_CONVEX_URL || process.env.CONVEX_URL if (!convexUrl) { throw new Error( "NEXT_PUBLIC_CONVEX_URL is required to sync manuals into Convex." ) } return convexUrl } function canonicalSiteVisibility(siteDomain: string) { const canonicalHost = canonicalizeTenantDomain( new URL(businessConfig.website).hostname ) return Array.from( new Set([ ...tenantDomainVariants(siteDomain), ...tenantDomainVariants(canonicalHost), ]) ) } async function uploadSelectedAssets( manuals: Awaited>, dryRun: boolean ) { const manualsRoot = getManualsFilesRoot() const thumbnailsRoot = getManualsThumbnailsRoot() const stats = { manualsUploaded: 0, manualsSkipped: 0, thumbnailsUploaded: 0, thumbnailsSkipped: 0, } for (const manual of manuals) { const manualKey = manual.path const manualPath = join(manualsRoot, manual.path) const shouldSkipManual = await manualAssetExists("manuals", manualKey) if (shouldSkipManual) { stats.manualsSkipped += 1 } else if (dryRun) { stats.manualsUploaded += 1 } else { await uploadManualAsset("manuals", manualKey, await readFile(manualPath)) stats.manualsUploaded += 1 } if (!manual.thumbnailUrl) { continue } const thumbnailKey = manual.thumbnailUrl const thumbnailPath = join(thumbnailsRoot, manual.thumbnailUrl) const shouldSkipThumbnail = await manualAssetExists( "thumbnails", thumbnailKey ) if (shouldSkipThumbnail) { stats.thumbnailsSkipped += 1 } else if (dryRun) { stats.thumbnailsUploaded += 1 } else { await uploadManualAsset( "thumbnails", thumbnailKey, await readFile(thumbnailPath) ) stats.thumbnailsUploaded += 1 } } return stats } function normalizeManualForConvex( manual: Awaited>[number], importBatch: string, siteDomain: string ) { const manualUrl = buildManualAssetUrl(manual.path) const thumbnailUrl = manual.thumbnailUrl ? buildThumbnailAssetUrl(manual.thumbnailUrl) : undefined return { filename: manual.filename, path: manual.path, manufacturer: manual.manufacturer, category: manual.category, size: manual.size, lastModified: manual.lastModified ? manual.lastModified.getTime() : undefined, searchTerms: manual.searchTerms, commonNames: manual.commonNames, thumbnailUrl, manualUrl, hasParts: Boolean( manual.searchTerms?.some((term) => term.toLowerCase().includes("parts")) ), assetSource: getManualsAssetSource(), sourcePath: manual.path, sourceSite: businessConfig.name, sourceDomain: siteDomain, siteVisibility: canonicalSiteVisibility(siteDomain), importBatch, } satisfies ManualUpsertInput } async function main() { const args = parseArgs(process.argv.slice(2)) const siteDomain = resolveManualsTenantDomain({ envTenantDomain: process.env.MANUALS_TENANT_DOMAIN, envSiteDomain: getSiteDomain(), }) if (!siteDomain) { throw new Error( "Could not resolve manuals tenant domain. Set MANUALS_TENANT_DOMAIN or NEXT_PUBLIC_SITE_DOMAIN." ) } const allManuals = await scanManuals() const selection = selectManualsForSite(allManuals, siteDomain) const selectedManuals = args.limit ? selection.manuals.slice(0, args.limit) : selection.manuals const manualsWithThumbnails = selectedManuals.filter((manual) => Boolean(manual.thumbnailUrl) ) const importBatch = new Date().toISOString() const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual, importBatch, siteDomain) ) console.log("[manuals-sync] selection", { siteDomain: selection.siteDomain, scanned: selection.totalInput, mdbEligible: selection.mdbEligible, siteVisible: selection.siteVisible, finalSelected: selection.finalCount, selectedForRun: selectedManuals.length, thumbnailsForRun: manualsWithThumbnails.length, assetSource: getManualsAssetSource(), }) if (!args.skipUpload) { if (!hasManualObjectStorageConfig()) { throw new Error("Manual object storage credentials are not configured.") } console.log("[manuals-sync] uploading selected manuals and thumbnails") const uploadStats = await uploadSelectedAssets(selectedManuals, args.dryRun) console.log("[manuals-sync] upload summary", uploadStats) } if (args.dryRun) { console.log("[manuals-sync] dry run complete") return } const convex = new ConvexHttpClient(readConvexUrl()) const batchSize = 100 let synced = 0 for (let index = 0; index < payload.length; index += batchSize) { const slice = payload.slice(index, index + batchSize) await convex.mutation(UPSERT_MANUALS, { manuals: slice }) synced += slice.length console.log("[manuals-sync] upserted batch", { synced, total: payload.length, }) } console.log("[manuals-sync] completed", { synced, importBatch, siteDomain, }) } main().catch((error) => { console.error("[manuals-sync] failed", error) process.exit(1) })