From 7f97f76ca124c70fdf9bf6268d58a8e2a5a3941f Mon Sep 17 00:00:00 2001 From: DMleadgen Date: Fri, 27 Mar 2026 12:39:33 -0600 Subject: [PATCH] Fix Rocky manuals catalog delivery --- app/api/manuals/[...path]/route.ts | 151 +++++++----------- app/api/thumbnails/[...path]/route.ts | 171 +++++++++----------- app/manuals/dashboard/page.tsx | 9 +- app/manuals/page.tsx | 34 +--- lib/manuals-object-storage.ts | 221 ++++++++++++++++++++++++++ lib/manuals-site-selection.ts | 50 ++++++ lib/manuals-storage.ts | 6 +- lib/manuals-types.ts | 67 +------- lib/site-manufacturer-mapping.json | 16 +- scripts/deploy-readiness.mjs | 12 +- scripts/sync-manuals-to-convex.ts | 122 +++++++++----- scripts/upload-to-r2.js | 41 +++-- 12 files changed, 560 insertions(+), 340 deletions(-) create mode 100644 lib/manuals-object-storage.ts create mode 100644 lib/manuals-site-selection.ts diff --git a/app/api/manuals/[...path]/route.ts b/app/api/manuals/[...path]/route.ts index 44bebbf0..f0821019 100644 --- a/app/api/manuals/[...path]/route.ts +++ b/app/api/manuals/[...path]/route.ts @@ -1,114 +1,81 @@ -import { NextRequest, NextResponse } from 'next/server' -import { readFile } from 'fs/promises' -import { join } from 'path' -import { existsSync } from 'fs' -import { getManualsFilesRoot } from '@/lib/manuals-paths' +import { NextResponse } from "next/server" +import { readFile } from "node:fs/promises" +import { existsSync } from "node:fs" +import { join } from "node:path" +import { getManualsFilesRoot } from "@/lib/manuals-paths" +import { getManualAssetFromStorage } from "@/lib/manuals-object-storage" -// API routes are not supported in static export (GHL hosting) -// Manuals are now served as static files from /manuals/ -export const dynamic = 'force-static' +export const dynamic = "force-dynamic" -/** - * API route to serve PDF manuals - * This allows serving files from outside the public folder - * - * Usage: /api/manuals/BevMax/manual.pdf - * NOTE: This route is disabled for static export. Use /manuals/ paths instead. - */ +function decodeSegments(pathArray: string[]) { + return pathArray.map((segment) => { + try { + let decoded = segment + while (decoded !== decodeURIComponent(decoded)) { + decoded = decodeURIComponent(decoded) + } + return decoded + } catch { + return segment + } + }) +} -// Required for static export - returns empty array to skip this route -export async function generateStaticParams(): Promise> { - return [] +function invalidPath(pathValue: string) { + return pathValue.includes("..") || pathValue.startsWith("/") } export async function GET( - request: NextRequest, - { params }: { params: Promise<{ path: string[] }> } + _request: Request, + { params }: { params: Promise<{ path: string[] }> }, ) { try { const { path: pathArray } = await params - - // Decode URL-encoded path segments - handle multiple encodings - const decodedPath = pathArray.map(segment => { - try { - // Try decoding multiple times in case of double encoding - let decoded = segment - while (decoded !== decodeURIComponent(decoded)) { - decoded = decodeURIComponent(decoded) - } - return decoded - } catch { - // If decoding fails, use original - return segment - } - }) - const filePath = decodedPath.join('/') - - // Security: Prevent directory traversal - if (filePath.includes('..') || filePath.startsWith('/')) { - return new NextResponse('Invalid path', { status: 400 }) + const decodedPath = decodeSegments(pathArray) + const filePath = decodedPath.join("/") + + if (invalidPath(filePath) || !filePath.toLowerCase().endsWith(".pdf")) { + return new NextResponse("Invalid path", { status: 400 }) } - - // Construct full path to manual + + const storageObject = await getManualAssetFromStorage("manuals", filePath) + if (storageObject) { + return new NextResponse(Buffer.from(storageObject.body), { + headers: { + "Content-Type": storageObject.contentType || "application/pdf", + "Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "manual.pdf")}"`, + "Cache-Control": "public, max-age=31536000, immutable", + "X-Content-Type-Options": "nosniff", + }, + }) + } + const manualsDir = getManualsFilesRoot() const fullPath = join(manualsDir, filePath) - - // Normalize paths to handle both forward and backward slashes - const normalizedFullPath = fullPath.replace(/\\/g, '/') - const normalizedManualsDir = manualsDir.replace(/\\/g, '/') - - // Verify file exists and is within manuals directory - let fileToRead = normalizedFullPath - if (!existsSync(normalizedFullPath)) { - // Try with original path in case of encoding issues - if (existsSync(fullPath)) { - fileToRead = fullPath - } else { - // Try with different path separators - const altPath = fullPath.replace(/\//g, '\\') - if (existsSync(altPath)) { - fileToRead = altPath - } else { - console.error(`File not found: ${normalizedFullPath}`) - console.error(`Also tried: ${fullPath}`) - return new NextResponse('File not found', { status: 404 }) - } - } + const normalizedFullPath = fullPath.replace(/\\/g, "/") + const normalizedManualsDir = manualsDir.replace(/\\/g, "/") + + if (!existsSync(normalizedFullPath) && !existsSync(fullPath)) { + return new NextResponse("File not found", { status: 404 }) } - - // Verify the file is within the manuals directory (security check) - const resolvedPath = fileToRead.replace(/\\/g, '/') - if (!resolvedPath.startsWith(normalizedManualsDir.replace(/\\/g, '/'))) { - console.error(`Security violation: Path outside manuals directory: ${resolvedPath}`) - return new NextResponse('Invalid path', { status: 400 }) + + const fileToRead = existsSync(normalizedFullPath) ? normalizedFullPath : fullPath + const resolvedPath = fileToRead.replace(/\\/g, "/") + if (!resolvedPath.startsWith(normalizedManualsDir)) { + return new NextResponse("Invalid path", { status: 400 }) } - - // Only allow PDF files - if (!filePath.toLowerCase().endsWith('.pdf')) { - return new NextResponse('Invalid file type', { status: 400 }) - } - - // Read and serve the file + const fileBuffer = await readFile(fileToRead) - - // Get filename for Content-Disposition header - const filename = decodedPath[decodedPath.length - 1] - return new NextResponse(fileBuffer, { headers: { - 'Content-Type': 'application/pdf', - 'Content-Disposition': `inline; filename="${encodeURIComponent(filename)}"`, - 'Cache-Control': 'public, max-age=31536000, immutable', - 'X-Content-Type-Options': 'nosniff', + "Content-Type": "application/pdf", + "Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "manual.pdf")}"`, + "Cache-Control": "public, max-age=31536000, immutable", + "X-Content-Type-Options": "nosniff", }, }) } catch (error) { - console.error('Error serving manual:', error) - if (error instanceof Error) { - console.error('Error details:', error.message, error.stack) - } - return new NextResponse('Internal server error', { status: 500 }) + console.error("[manuals-api] failed to serve manual", error) + return new NextResponse("Internal server error", { status: 500 }) } } - - diff --git a/app/api/thumbnails/[...path]/route.ts b/app/api/thumbnails/[...path]/route.ts index 8e3968c7..cfd2e8d2 100644 --- a/app/api/thumbnails/[...path]/route.ts +++ b/app/api/thumbnails/[...path]/route.ts @@ -1,118 +1,97 @@ -import { NextRequest, NextResponse } from 'next/server' -import { readFile } from 'fs/promises' -import { existsSync } from 'fs' -import { join } from 'path' -import { getManualsThumbnailsRoot } from '@/lib/manuals-paths' +import { NextResponse } from "next/server" +import { readFile } from "node:fs/promises" +import { existsSync } from "node:fs" +import { join } from "node:path" +import { getManualsThumbnailsRoot } from "@/lib/manuals-paths" +import { getManualAssetFromStorage } from "@/lib/manuals-object-storage" -// API routes are not supported in static export (GHL hosting) -// Thumbnails are now served as static files from /thumbnails/ -export const dynamic = 'force-static' +export const dynamic = "force-dynamic" -/** - * API route to serve thumbnail images for PDF manuals - * This allows serving files from outside the public folder - * - * Usage: /api/thumbnails/BevMax/manual.jpg - * NOTE: This route is disabled for static export. Use /thumbnails/ paths instead. - */ +function decodeSegments(pathArray: string[]) { + return pathArray.map((segment) => { + try { + let decoded = segment + while (decoded !== decodeURIComponent(decoded)) { + decoded = decodeURIComponent(decoded) + } + return decoded + } catch { + return segment + } + }) +} -// Required for static export - returns empty array to skip this route -export async function generateStaticParams(): Promise> { - return [] +function invalidPath(pathValue: string) { + return pathValue.includes("..") || pathValue.startsWith("/") +} + +function getContentType(pathValue: string) { + if (pathValue.toLowerCase().endsWith(".png")) { + return "image/png" + } + + if (pathValue.toLowerCase().endsWith(".webp")) { + return "image/webp" + } + + return "image/jpeg" } export async function GET( - request: NextRequest, - { params }: { params: Promise<{ path: string[] }> } + _request: Request, + { params }: { params: Promise<{ path: string[] }> }, ) { try { const { path: pathArray } = await params - - // Decode URL-encoded path segments - handle multiple encodings - const decodedPath = pathArray.map(segment => { - try { - // Try decoding multiple times in case of double encoding - let decoded = segment - while (decoded !== decodeURIComponent(decoded)) { - decoded = decodeURIComponent(decoded) - } - return decoded - } catch { - // If decoding fails, use original - return segment - } - }) - const filePath = decodedPath.join('/') - - // Security: Prevent directory traversal - if (filePath.includes('..') || filePath.startsWith('/')) { - return new NextResponse('Invalid path', { status: 400 }) + const decodedPath = decodeSegments(pathArray) + const filePath = decodedPath.join("/") + const lowerPath = filePath.toLowerCase() + + if ( + invalidPath(filePath) || + (!lowerPath.endsWith(".jpg") && !lowerPath.endsWith(".jpeg") && !lowerPath.endsWith(".png") && !lowerPath.endsWith(".webp")) + ) { + return new NextResponse("Invalid path", { status: 400 }) } - - // Construct full path to thumbnail + + const storageObject = await getManualAssetFromStorage("thumbnails", filePath) + if (storageObject) { + return new NextResponse(Buffer.from(storageObject.body), { + headers: { + "Content-Type": storageObject.contentType || getContentType(filePath), + "Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "thumbnail.jpg")}"`, + "Cache-Control": "public, max-age=31536000, immutable", + "X-Content-Type-Options": "nosniff", + }, + }) + } + const thumbnailsDir = getManualsThumbnailsRoot() const fullPath = join(thumbnailsDir, filePath) - - // Normalize paths to handle both forward and backward slashes - const normalizedFullPath = fullPath.replace(/\\/g, '/') - const normalizedThumbnailsDir = thumbnailsDir.replace(/\\/g, '/') - - // Verify file exists and is within thumbnails directory - let fileToRead = normalizedFullPath - if (!existsSync(normalizedFullPath)) { - // Try with original path in case of encoding issues - if (existsSync(fullPath)) { - fileToRead = fullPath - } else { - // Try with different path separators - const altPath = fullPath.replace(/\//g, '\\') - if (existsSync(altPath)) { - fileToRead = altPath - } else { - // Thumbnail doesn't exist - return 404 (this is expected for manuals without thumbnails) - return new NextResponse('Thumbnail not found', { status: 404 }) - } - } + const normalizedFullPath = fullPath.replace(/\\/g, "/") + const normalizedThumbnailsDir = thumbnailsDir.replace(/\\/g, "/") + + if (!existsSync(normalizedFullPath) && !existsSync(fullPath)) { + return new NextResponse("Thumbnail not found", { status: 404 }) } - - // Verify the file is within the thumbnails directory (security check) - const resolvedPath = fileToRead.replace(/\\/g, '/') - if (!resolvedPath.startsWith(normalizedThumbnailsDir.replace(/\\/g, '/'))) { - console.error(`Security violation: Path outside thumbnails directory: ${resolvedPath}`) - return new NextResponse('Invalid path', { status: 400 }) + + const fileToRead = existsSync(normalizedFullPath) ? normalizedFullPath : fullPath + const resolvedPath = fileToRead.replace(/\\/g, "/") + if (!resolvedPath.startsWith(normalizedThumbnailsDir)) { + return new NextResponse("Invalid path", { status: 400 }) } - - // Only allow image files (jpg, jpeg, png) - const lowerPath = filePath.toLowerCase() - if (!lowerPath.endsWith('.jpg') && !lowerPath.endsWith('.jpeg') && !lowerPath.endsWith('.png')) { - return new NextResponse('Invalid file type', { status: 400 }) - } - - // Read and serve the file + const fileBuffer = await readFile(fileToRead) - - // Get filename for Content-Disposition header - const filename = decodedPath[decodedPath.length - 1] - - // Determine content type - const contentType = lowerPath.endsWith('.png') - ? 'image/png' - : 'image/jpeg' - return new NextResponse(fileBuffer, { headers: { - 'Content-Type': contentType, - 'Content-Disposition': `inline; filename="${encodeURIComponent(filename)}"`, - 'Cache-Control': 'public, max-age=31536000, immutable', - 'X-Content-Type-Options': 'nosniff', + "Content-Type": getContentType(filePath), + "Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "thumbnail.jpg")}"`, + "Cache-Control": "public, max-age=31536000, immutable", + "X-Content-Type-Options": "nosniff", }, }) } catch (error) { - console.error('Error serving thumbnail:', error) - if (error instanceof Error) { - console.error('Error details:', error.message, error.stack) - } - return new NextResponse('Internal server error', { status: 500 }) + console.error("[manuals-api] failed to serve thumbnail", error) + return new NextResponse("Internal server error", { status: 500 }) } } - diff --git a/app/manuals/dashboard/page.tsx b/app/manuals/dashboard/page.tsx index 6ab0b4f4..ee2768c0 100644 --- a/app/manuals/dashboard/page.tsx +++ b/app/manuals/dashboard/page.tsx @@ -2,12 +2,20 @@ import { Metadata } from 'next' import { ManualsDashboardClient } from '@/components/manuals-dashboard-client' import { PublicInset, PublicPageHeader } from '@/components/public-surface' import { getConvexManualDashboard } from '@/lib/convex-service' +import { businessConfig } from '@/lib/seo-config' import { readFileSync } from 'fs' import { join } from 'path' export const metadata: Metadata = { title: 'Manual Processing Dashboard | Rocky Mountain Vending', description: 'Comprehensive dashboard showing processed manual data, statistics, and analytics', + alternates: { + canonical: `${businessConfig.website}/manuals/dashboard`, + }, + robots: { + index: false, + follow: false, + }, } interface DashboardData { @@ -99,4 +107,3 @@ export default async function ManualsDashboardPage() { - diff --git a/app/manuals/page.tsx b/app/manuals/page.tsx index 9dfa7e07..81eab48a 100644 --- a/app/manuals/page.tsx +++ b/app/manuals/page.tsx @@ -10,17 +10,8 @@ import { groupManuals, getManufacturers, getCategories, - filterMDBManuals, - filterManualsBySite, - filterManufacturersByMinCount, } from '@/lib/manuals' -import { - getSiteDomain, - getAllowedManufacturers, - getMinManualCount, - getManufacturerAliases, - shouldIncludePaymentComponents, -} from '@/lib/site-config' +import { selectManualsForSite } from '@/lib/manuals-site-selection' import { generateStructuredData } from '@/lib/seo' import { getManualsThumbnailsRoot } from '@/lib/manuals-paths' @@ -84,31 +75,10 @@ export const metadata: Metadata = { } export default async function ManualsPage() { - // Get site configuration - const siteDomain = getSiteDomain() - const allowedManufacturers = getAllowedManufacturers(siteDomain) - const minManualCount = getMinManualCount(siteDomain) - const manufacturerAliases = getManufacturerAliases() - const includePaymentComponents = shouldIncludePaymentComponents(siteDomain) - // Prefer Convex-backed metadata, but keep filesystem fallback in place until the shared catalog is fully populated. const convexManuals = await listConvexManuals() const allManuals = convexManuals.length > 0 ? convexManuals : await scanManuals() - - // Filtering chain: - // 1. Filter by MDB (existing) - let manuals = filterMDBManuals(allManuals) - - // 2. Filter by site tier/allowed manufacturers (new) - manuals = filterManualsBySite( - manuals, - allowedManufacturers, - manufacturerAliases, - includePaymentComponents - ) - - // 3. Filter by minimum manual count per manufacturer (new) - manuals = filterManufacturersByMinCount(manuals, minManualCount) + let manuals = selectManualsForSite(allManuals).manuals // Hide broken local thumbnails so the public manuals page doesn't spam 404s. const thumbnailsRoot = getManualsThumbnailsRoot() diff --git a/lib/manuals-object-storage.ts b/lib/manuals-object-storage.ts new file mode 100644 index 00000000..32869014 --- /dev/null +++ b/lib/manuals-object-storage.ts @@ -0,0 +1,221 @@ +import { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3" + +type AssetKind = "manuals" | "thumbnails" + +type StorageConfig = { + endpoint: string + region: string + accessKeyId: string + secretAccessKey: string + forcePathStyle: boolean + disableSse: boolean + disableChecksums: boolean + manualsBucket: string + thumbnailsBucket: string +} + +type StoredObject = { + body: Uint8Array + contentType?: string + contentLength?: number + lastModified?: Date + etag?: string +} + +let cachedClient: S3Client | null = null +let cachedSignature: string | null = null + +function readOptionalEnv(...keys: string[]) { + for (const key of keys) { + const value = String(process.env[key] ?? "").trim() + if (value) { + return value + } + } + + return "" +} + +function readBooleanEnv(keys: string[], fallback: boolean) { + for (const key of keys) { + const value = String(process.env[key] ?? "").trim().toLowerCase() + if (!value) { + continue + } + + if (["1", "true", "yes", "on"].includes(value)) { + return true + } + + if (["0", "false", "no", "off"].includes(value)) { + return false + } + } + + return fallback +} + +function getStorageConfig(): StorageConfig | null { + const endpoint = readOptionalEnv("MANUALS_STORAGE_ENDPOINT", "S3_ENDPOINT_URL", "CLOUDFLARE_R2_ENDPOINT") + const accessKeyId = readOptionalEnv( + "MANUALS_STORAGE_ACCESS_KEY_ID", + "CLOUDFLARE_R2_ACCESS_KEY_ID", + "AWS_ACCESS_KEY_ID", + "AWS_ACCESS_KEY", + ) + const secretAccessKey = readOptionalEnv( + "MANUALS_STORAGE_SECRET_ACCESS_KEY", + "CLOUDFLARE_R2_SECRET_ACCESS_KEY", + "AWS_SECRET_ACCESS_KEY", + "AWS_SECRET_KEY", + ) + const region = readOptionalEnv("MANUALS_STORAGE_REGION", "AWS_REGION") || "us-east-1" + const manualsBucket = readOptionalEnv("MANUALS_STORAGE_MANUALS_BUCKET", "R2_MANUALS_BUCKET") + const thumbnailsBucket = readOptionalEnv("MANUALS_STORAGE_THUMBNAILS_BUCKET", "R2_THUMBNAILS_BUCKET") + + if (!endpoint || !accessKeyId || !secretAccessKey || !manualsBucket || !thumbnailsBucket) { + return null + } + + return { + endpoint, + region, + accessKeyId, + secretAccessKey, + forcePathStyle: readBooleanEnv(["MANUALS_STORAGE_FORCE_PATH_STYLE", "AWS_S3_FORCE_PATH_STYLE"], true), + disableSse: readBooleanEnv(["MANUALS_STORAGE_DISABLE_SSE", "AWS_S3_DISABLE_SSE"], true), + disableChecksums: readBooleanEnv( + ["MANUALS_STORAGE_DISABLE_CHECKSUMS", "AWS_S3_DISABLE_CHECKSUMS"], + true, + ), + manualsBucket, + thumbnailsBucket, + } +} + +function getBucketName(kind: AssetKind, config: StorageConfig) { + return kind === "manuals" ? config.manualsBucket : config.thumbnailsBucket +} + +function getClient(config: StorageConfig) { + const signature = JSON.stringify(config) + if (!cachedClient || cachedSignature !== signature) { + cachedClient = new S3Client({ + region: config.region, + endpoint: config.endpoint, + forcePathStyle: config.forcePathStyle, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) + cachedSignature = signature + } + + return cachedClient +} + +function guessContentType(key: string) { + const lower = key.toLowerCase() + if (lower.endsWith(".pdf")) { + return "application/pdf" + } + + if (lower.endsWith(".png")) { + return "image/png" + } + + if (lower.endsWith(".webp")) { + return "image/webp" + } + + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) { + return "image/jpeg" + } + + return "application/octet-stream" +} + +export function hasManualObjectStorageConfig() { + return getStorageConfig() !== null +} + +export async function getManualAssetFromStorage(kind: AssetKind, key: string): Promise { + const config = getStorageConfig() + if (!config) { + return null + } + + try { + const response = await getClient(config).send( + new GetObjectCommand({ + Bucket: getBucketName(kind, config), + Key: key, + }), + ) + + if (!response.Body) { + return null + } + + return { + body: await response.Body.transformToByteArray(), + contentType: response.ContentType, + contentLength: response.ContentLength, + lastModified: response.LastModified, + etag: response.ETag, + } + } catch (error) { + const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode + if (statusCode === 404 || statusCode === 403) { + return null + } + + throw error + } +} + +export async function manualAssetExists(kind: AssetKind, key: string) { + const config = getStorageConfig() + if (!config) { + return false + } + + try { + await getClient(config).send( + new HeadObjectCommand({ + Bucket: getBucketName(kind, config), + Key: key, + }), + ) + return true + } catch (error) { + const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode + if (statusCode === 404) { + return false + } + + throw error + } +} + +export async function uploadManualAsset( + kind: AssetKind, + key: string, + body: Uint8Array, + contentType?: string, +) { + const config = getStorageConfig() + if (!config) { + throw new Error("Manual object storage is not configured.") + } + + await getClient(config).send( + new PutObjectCommand({ + Bucket: getBucketName(kind, config), + Key: key, + Body: body, + ContentType: contentType || guessContentType(key), + }), + ) +} diff --git a/lib/manuals-site-selection.ts b/lib/manuals-site-selection.ts new file mode 100644 index 00000000..01b4391d --- /dev/null +++ b/lib/manuals-site-selection.ts @@ -0,0 +1,50 @@ +import type { Manual } from "@/lib/manuals-types" +import { + filterMDBManuals, + filterManualsBySite, + filterManufacturersByMinCount, +} from "@/lib/manuals" +import { + getAllowedManufacturers, + getManufacturerAliases, + getMinManualCount, + getSiteDomain, + shouldIncludePaymentComponents, +} from "@/lib/site-config" + +export type ManualSelectionResult = { + siteDomain: string + totalInput: number + mdbEligible: number + siteVisible: number + finalCount: number + manuals: Manual[] +} + +export function selectManualsForSite( + manuals: Manual[], + siteDomain: string = getSiteDomain(), +): ManualSelectionResult { + const allowedManufacturers = getAllowedManufacturers(siteDomain) + const manufacturerAliases = getManufacturerAliases() + const minManualCount = getMinManualCount(siteDomain) + const includePaymentComponents = shouldIncludePaymentComponents(siteDomain) + + const mdbEligible = filterMDBManuals(manuals) + const siteVisible = filterManualsBySite( + mdbEligible, + allowedManufacturers, + manufacturerAliases, + includePaymentComponents, + ) + const finalManuals = filterManufacturersByMinCount(siteVisible, minManualCount) + + return { + siteDomain, + totalInput: manuals.length, + mdbEligible: mdbEligible.length, + siteVisible: siteVisible.length, + finalCount: finalManuals.length, + manuals: finalManuals, + } +} diff --git a/lib/manuals-storage.ts b/lib/manuals-storage.ts index d586a406..02eed9fd 100644 --- a/lib/manuals-storage.ts +++ b/lib/manuals-storage.ts @@ -13,17 +13,17 @@ function getSiteBaseUrl() { } export function getManualsAssetBaseUrl() { - return cleanBaseUrl(process.env.NEXT_PUBLIC_MANUALS_BASE_URL || `${getSiteBaseUrl()}/manuals`) + return cleanBaseUrl(process.env.NEXT_PUBLIC_MANUALS_BASE_URL || `${getSiteBaseUrl()}/api/manuals`) } export function getThumbnailsAssetBaseUrl() { - return cleanBaseUrl(process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL || `${getSiteBaseUrl()}/thumbnails`) + return cleanBaseUrl(process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL || `${getSiteBaseUrl()}/api/thumbnails`) } export function getManualsAssetSource() { return process.env.NEXT_PUBLIC_MANUALS_BASE_URL && process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL ? "external-object-storage" - : "site-static" + : "site-proxy" } export function buildManualAssetUrl(relativePath: string) { diff --git a/lib/manuals-types.ts b/lib/manuals-types.ts index 5e274b4d..3bb2bed6 100644 --- a/lib/manuals-types.ts +++ b/lib/manuals-types.ts @@ -1,3 +1,5 @@ +import { buildManualAssetUrl, buildThumbnailAssetUrl } from "@/lib/manuals-storage" + export interface Manual { filename: string path: string @@ -46,40 +48,7 @@ export function getManualUrl(manual: Manual): string { relativePath = manual.path } - // Encode each path segment to handle special characters (spaces, %, etc.) - // Split by '/' and encode each segment individually, then rejoin with '/' - // This preserves the path structure while encoding special characters - const pathSegments = relativePath.split('/') - const encodedSegments = pathSegments.map(segment => { - // Only encode if the segment contains characters that need encoding - // This avoids double-encoding already encoded segments - try { - const decoded = decodeURIComponent(segment) - // If decoding succeeds and is different, it was already encoded - if (decoded !== segment) { - return segment // Keep as-is if already encoded - } - } catch { - // If decoding fails, it's not encoded, so encode it - } - return encodeURIComponent(segment) - }) - const encodedPath = encodedSegments.join('/') - - // Check for external base URL (e.g., Cloudflare) - const baseUrl = process.env.NEXT_PUBLIC_MANUALS_BASE_URL - if (baseUrl) { - // Remove trailing slash from base URL if present - const cleanBaseUrl = baseUrl.replace(/\/$/, '') - return `${cleanBaseUrl}/${encodedPath}` - } - - if (process.env.NODE_ENV === 'development') { - return `/api/manuals/${encodedPath}` - } - - // Use local static path for exported hosting - return `/manuals/${encodedPath}` + return buildManualAssetUrl(relativePath) } /** @@ -109,33 +78,5 @@ export function getThumbnailUrl(manual: Manual): string | null { relativePath = manual.thumbnailUrl } - // Encode each path segment to handle special characters - const pathSegments = relativePath.split('/') - const encodedSegments = pathSegments.map(segment => { - try { - const decoded = decodeURIComponent(segment) - if (decoded !== segment) { - return segment // Keep as-is if already encoded - } - } catch { - // If decoding fails, it's not encoded, so encode it - } - return encodeURIComponent(segment) - }) - const encodedPath = encodedSegments.join('/') - - // Check for external base URL (e.g., Cloudflare) - const baseUrl = process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL - if (baseUrl) { - // Remove trailing slash from base URL if present - const cleanBaseUrl = baseUrl.replace(/\/$/, '') - return `${cleanBaseUrl}/${encodedPath}` - } - - if (process.env.NODE_ENV === 'development') { - return `/api/thumbnails/${encodedPath}` - } - - // Use local static path for exported hosting - return `/thumbnails/${encodedPath}` + return buildThumbnailAssetUrl(relativePath) } diff --git a/lib/site-manufacturer-mapping.json b/lib/site-manufacturer-mapping.json index 19f86f86..ffd34cf4 100644 --- a/lib/site-manufacturer-mapping.json +++ b/lib/site-manufacturer-mapping.json @@ -15,6 +15,21 @@ "includePaymentComponents": true, "minManualCount": 3 }, + "rmv.abundancepartners.app": { + "tier": 1, + "description": "Rocky Mountain Vending staging host mirrors the production Rocky manuals subset", + "manufacturers": [ + "Crane", + "Royal Vendors", + "GPL", + "AP", + "Dixie-Narco", + "USI", + "Vendo" + ], + "includePaymentComponents": true, + "minManualCount": 3 + }, "vending.support": { "tier": 2, "description": "Site with medium manufacturer count - includes Tier 1 plus additional popular manufacturers", @@ -73,4 +88,3 @@ } - diff --git a/scripts/deploy-readiness.mjs b/scripts/deploy-readiness.mjs index dd027495..bc0785b5 100644 --- a/scripts/deploy-readiness.mjs +++ b/scripts/deploy-readiness.mjs @@ -15,7 +15,7 @@ const REQUIRED_ENV_GROUPS = [ }, { label: "Manual asset delivery", - keys: ["NEXT_PUBLIC_MANUALS_BASE_URL", "NEXT_PUBLIC_THUMBNAILS_BASE_URL", "CLOUDFLARE_R2_ENDPOINT", "R2_MANUALS_BUCKET", "R2_THUMBNAILS_BUCKET"], + keys: ["R2_MANUALS_BUCKET", "R2_THUMBNAILS_BUCKET"], }, { label: "Admin and auth", @@ -109,6 +109,14 @@ function hasManualStorageCredentials() { ].every(Boolean) } +function hasManualStorageEndpoint() { + return Boolean( + readValue("MANUALS_STORAGE_ENDPOINT") || + readValue("S3_ENDPOINT_URL") || + readValue("CLOUDFLARE_R2_ENDPOINT"), + ) +} + function heading(title) { console.log(`\n== ${title} ==`) } @@ -189,6 +197,8 @@ function main() { if (!hasManualStorageCredentials()) { failures.push("Manual asset storage credentials are incomplete. Set R2/S3 access key and secret env vars before release.") + } else if (!hasManualStorageEndpoint()) { + failures.push("Manual asset storage endpoint is incomplete. Set S3_ENDPOINT_URL or CLOUDFLARE_R2_ENDPOINT before release.") } else { console.log("Manual asset storage credentials: present") } diff --git a/scripts/sync-manuals-to-convex.ts b/scripts/sync-manuals-to-convex.ts index 7c6f428f..da961d27 100644 --- a/scripts/sync-manuals-to-convex.ts +++ b/scripts/sync-manuals-to-convex.ts @@ -1,11 +1,19 @@ -import { spawnSync } from "node:child_process" +import { readFile } from "node:fs/promises" +import { join } from "node:path" import { config as loadEnv } from "dotenv" import { ConvexHttpClient } from "convex/browser" import { makeFunctionReference } from "convex/server" import { scanManuals } from "../lib/manuals" -import { buildManualAssetUrl, buildThumbnailAssetUrl, getManualsAssetSource } from "../lib/manuals-storage" +import { getManualsAssetSource, buildManualAssetUrl, buildThumbnailAssetUrl } from "../lib/manuals-storage" import { businessConfig } from "../lib/seo-config" import { getSiteDomain } from "../lib/site-config" +import { selectManualsForSite } from "../lib/manuals-site-selection" +import { getManualsFilesRoot, getManualsThumbnailsRoot } from "../lib/manuals-paths" +import { + hasManualObjectStorageConfig, + manualAssetExists, + uploadManualAsset, +} from "../lib/manuals-object-storage" loadEnv({ path: ".env.local" }) loadEnv({ path: ".env.staging", override: false }) @@ -52,35 +60,64 @@ function readConvexUrl() { return convexUrl } -function shouldUploadAssets() { - const accessKey = - process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || - process.env.AWS_ACCESS_KEY_ID || - process.env.AWS_ACCESS_KEY - const secret = - process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY || - process.env.AWS_SECRET_ACCESS_KEY || - process.env.AWS_SECRET_KEY - - return Boolean(accessKey && secret) +function canonicalSiteVisibility(siteDomain: string) { + const canonicalHost = new URL(businessConfig.website).hostname + return Array.from(new Set([siteDomain, canonicalHost])) } -function runAssetUpload() { - const result = spawnSync("node", ["scripts/upload-to-r2.js", "--type", "all", "--incremental"], { - cwd: process.cwd(), - stdio: "inherit", - }) - - if ((result.status ?? 1) !== 0) { - throw new Error("R2 upload step failed.") +async function uploadSelectedAssets( + manuals: Awaited>, + dryRun: boolean, +) { + const manualsRoot = getManualsFilesRoot() + const thumbnailsRoot = getManualsThumbnailsRoot() + const stats = { + manualsUploaded: 0, + manualsSkipped: 0, + thumbnailsUploaded: 0, + thumbnailsSkipped: 0, } + + for (const manual of manuals) { + const manualKey = manual.path + const manualPath = join(manualsRoot, manual.path) + const shouldSkipManual = await manualAssetExists("manuals", manualKey) + + if (shouldSkipManual) { + stats.manualsSkipped += 1 + } else if (dryRun) { + stats.manualsUploaded += 1 + } else { + await uploadManualAsset("manuals", manualKey, await readFile(manualPath)) + stats.manualsUploaded += 1 + } + + if (!manual.thumbnailUrl) { + continue + } + + const thumbnailKey = manual.thumbnailUrl + const thumbnailPath = join(thumbnailsRoot, manual.thumbnailUrl) + const shouldSkipThumbnail = await manualAssetExists("thumbnails", thumbnailKey) + + if (shouldSkipThumbnail) { + stats.thumbnailsSkipped += 1 + } else if (dryRun) { + stats.thumbnailsUploaded += 1 + } else { + await uploadManualAsset("thumbnails", thumbnailKey, await readFile(thumbnailPath)) + stats.thumbnailsUploaded += 1 + } + } + + return stats } function normalizeManualForConvex( manual: Awaited>[number], importBatch: string, + siteDomain: string, ) { - const siteDomain = getSiteDomain() const manualUrl = buildManualAssetUrl(manual.path) const thumbnailUrl = manual.thumbnailUrl ? buildThumbnailAssetUrl(manual.thumbnailUrl) : undefined @@ -100,32 +137,42 @@ function normalizeManualForConvex( sourcePath: manual.path, sourceSite: businessConfig.name, sourceDomain: siteDomain, - siteVisibility: [siteDomain], + siteVisibility: canonicalSiteVisibility(siteDomain), importBatch, } satisfies ManualUpsertInput } async function main() { const args = parseArgs(process.argv.slice(2)) - - if (!args.skipUpload && shouldUploadAssets()) { - console.log("[manuals-sync] uploading new manuals and thumbnails to object storage") - runAssetUpload() - } else if (!args.skipUpload) { - console.log("[manuals-sync] skipping asset upload because Cloudflare R2 credentials are not configured") - } - - const manuals = await scanManuals() - const selectedManuals = args.limit ? manuals.slice(0, args.limit) : manuals + const siteDomain = getSiteDomain() + const allManuals = await scanManuals() + const selection = selectManualsForSite(allManuals, siteDomain) + const selectedManuals = args.limit ? selection.manuals.slice(0, args.limit) : selection.manuals + const manualsWithThumbnails = selectedManuals.filter((manual) => Boolean(manual.thumbnailUrl)) const importBatch = new Date().toISOString() - const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual, importBatch)) + const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual, importBatch, siteDomain)) - console.log("[manuals-sync] scanned manuals", { - total: manuals.length, - selected: selectedManuals.length, + console.log("[manuals-sync] selection", { + siteDomain: selection.siteDomain, + scanned: selection.totalInput, + mdbEligible: selection.mdbEligible, + siteVisible: selection.siteVisible, + finalSelected: selection.finalCount, + selectedForRun: selectedManuals.length, + thumbnailsForRun: manualsWithThumbnails.length, assetSource: getManualsAssetSource(), }) + if (!args.skipUpload) { + if (!hasManualObjectStorageConfig()) { + throw new Error("Manual object storage credentials are not configured.") + } + + console.log("[manuals-sync] uploading selected manuals and thumbnails") + const uploadStats = await uploadSelectedAssets(selectedManuals, args.dryRun) + console.log("[manuals-sync] upload summary", uploadStats) + } + if (args.dryRun) { console.log("[manuals-sync] dry run complete") return @@ -148,6 +195,7 @@ async function main() { console.log("[manuals-sync] completed", { synced, importBatch, + siteDomain, }) } diff --git a/scripts/upload-to-r2.js b/scripts/upload-to-r2.js index eb558b9b..97a1cd7a 100644 --- a/scripts/upload-to-r2.js +++ b/scripts/upload-to-r2.js @@ -1,7 +1,7 @@ #!/usr/bin/env node /** - * Upload scripts to Cloudflare R2 using S3-compatible API + * Upload manuals assets to any S3-compatible object storage (Cloudflare R2, MinIO, etc.) * * Usage: * node scripts/upload-to-r2.js --type manuals --dry-run @@ -9,10 +9,8 @@ * node scripts/upload-to-r2.js --type all * * Environment variables required: - * CLOUDFLARE_R2_ACCESS_KEY_ID - * CLOUDFLARE_R2_SECRET_ACCESS_KEY - * CLOUDFLARE_R2_ENDPOINT (or uses default with account ID) - * CLOUDFLARE_ACCOUNT_ID + * S3_ENDPOINT_URL or CLOUDFLARE_R2_ENDPOINT + * AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* variants) */ import { S3Client, PutObjectCommand, ListObjectsV2Command, HeadObjectCommand } from '@aws-sdk/client-s3'; @@ -21,10 +19,25 @@ import { join, relative, dirname } from 'path'; import { existsSync } from 'fs'; const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || 'bd6f76304a840ba11b75f9ced84264f4'; -const ENDPOINT = process.env.CLOUDFLARE_R2_ENDPOINT || `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`; -const ACCESS_KEY_ID = process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY; +const ENDPOINT = + process.env.MANUALS_STORAGE_ENDPOINT || + process.env.S3_ENDPOINT_URL || + process.env.CLOUDFLARE_R2_ENDPOINT || + `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`; +const ACCESS_KEY_ID = + process.env.MANUALS_STORAGE_ACCESS_KEY_ID || + process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || + process.env.AWS_ACCESS_KEY_ID || + process.env.AWS_ACCESS_KEY; const SECRET_ACCESS_KEY = - process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY || process.env.AWS_SECRET_KEY; + process.env.MANUALS_STORAGE_SECRET_ACCESS_KEY || + process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY || + process.env.AWS_SECRET_ACCESS_KEY || + process.env.AWS_SECRET_KEY; +const REGION = process.env.MANUALS_STORAGE_REGION || process.env.AWS_REGION || 'us-east-1'; +const FORCE_PATH_STYLE = ['1', 'true', 'yes', 'on'].includes( + String(process.env.MANUALS_STORAGE_FORCE_PATH_STYLE || process.env.AWS_S3_FORCE_PATH_STYLE || 'true').toLowerCase() +); // Bucket names const MANUALS_BUCKET = process.env.R2_MANUALS_BUCKET || 'vending-vm-manuals'; @@ -46,16 +59,17 @@ const uploadType = typeArg || 'all'; const dryRun = args.includes('--dry-run') || args.includes('-d'); const incremental = args.includes('--incremental') || args.includes('-i'); -if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY) { - console.error('❌ Error: CLOUDFLARE_R2_ACCESS_KEY_ID and CLOUDFLARE_R2_SECRET_ACCESS_KEY must be set'); - console.error(' Set these in your .env file or environment variables'); +if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY || !ENDPOINT) { + console.error('❌ Error: S3-compatible manuals storage env vars are incomplete'); + console.error(' Set S3_ENDPOINT_URL plus AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* equivalents)'); process.exit(1); } // Initialize S3 client const s3Client = new S3Client({ - region: 'auto', + region: REGION, endpoint: ENDPOINT, + forcePathStyle: FORCE_PATH_STYLE, credentials: { accessKeyId: ACCESS_KEY_ID, secretAccessKey: SECRET_ACCESS_KEY, @@ -206,8 +220,7 @@ async function uploadDirectory(sourceDir, bucket, bucketName, dryRun = false, in * Main upload function */ async function main() { - console.log('🚀 Cloudflare R2 Upload Script'); - console.log(` Account ID: ${ACCOUNT_ID}`); + console.log('🚀 Manuals Object Storage Upload Script'); console.log(` Endpoint: ${ENDPOINT}`); console.log(` Mode: ${dryRun ? 'DRY RUN' : 'UPLOAD'}`); console.log(` Incremental: ${incremental ? 'Yes' : 'No'}`);