matt/Rocky_Mountain_Vending
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix Rocky manuals catalog delivery

Browse source
This commit is contained in:
DMleadgen 2026-03-27 12:39:33 -06:00
parent 6775ba0e93
commit 7f97f76ca1
Signed by: matt
GPG key ID: C2720CF8CD701894
12 changed files with 560 additions and 340 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

151
app/api/manuals/[...path]/route.ts
View file

@ -1,114 +1,81 @@
import { NextRequest, NextResponse } from 'next/server' import { NextResponse } from "next/server"
import { readFile } from 'fs/promises' import { readFile } from "node:fs/promises"
import { join } from 'path' import { existsSync } from "node:fs"
import { existsSync } from 'fs' import { join } from "node:path"
import { getManualsFilesRoot } from '@/lib/manuals-paths' import { getManualsFilesRoot } from "@/lib/manuals-paths"
import { getManualAssetFromStorage } from "@/lib/manuals-object-storage"
// API routes are not supported in static export (GHL hosting) export const dynamic = "force-dynamic"
// Manuals are now served as static files from /manuals/
export const dynamic = 'force-static'
/** function decodeSegments(pathArray: string[]) {
* API route to serve PDF manuals return pathArray.map((segment) => {
* This allows serving files from outside the public folder try {
* let decoded = segment
* Usage: /api/manuals/BevMax/manual.pdf while (decoded !== decodeURIComponent(decoded)) {
* NOTE: This route is disabled for static export. Use /manuals/ paths instead. decoded = decodeURIComponent(decoded)
*/ }
return decoded
} catch {
return segment
}
})
}
// Required for static export - returns empty array to skip this route function invalidPath(pathValue: string) {
export async function generateStaticParams(): Promise<Array<{ path: string[] }>> { return pathValue.includes("..") || pathValue.startsWith("/")
return []
} }
export async function GET( export async function GET(
request: NextRequest, _request: Request,
{ params }: { params: Promise<{ path: string[] }> } { params }: { params: Promise<{ path: string[] }> },
) { ) {
try { try {
const { path: pathArray } = await params const { path: pathArray } = await params
const decodedPath = decodeSegments(pathArray)
// Decode URL-encoded path segments - handle multiple encodings const filePath = decodedPath.join("/")
const decodedPath = pathArray.map(segment => {
try { if (invalidPath(filePath) || !filePath.toLowerCase().endsWith(".pdf")) {
// Try decoding multiple times in case of double encoding return new NextResponse("Invalid path", { status: 400 })
let decoded = segment
while (decoded !== decodeURIComponent(decoded)) {
decoded = decodeURIComponent(decoded)
}
return decoded
} catch {
// If decoding fails, use original
return segment
}
})
const filePath = decodedPath.join('/')
// Security: Prevent directory traversal
if (filePath.includes('..') || filePath.startsWith('/')) {
return new NextResponse('Invalid path', { status: 400 })
} }
// Construct full path to manual const storageObject = await getManualAssetFromStorage("manuals", filePath)
if (storageObject) {
return new NextResponse(Buffer.from(storageObject.body), {
headers: {
"Content-Type": storageObject.contentType || "application/pdf",
"Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "manual.pdf")}"`,
"Cache-Control": "public, max-age=31536000, immutable",
"X-Content-Type-Options": "nosniff",
},
})
}
const manualsDir = getManualsFilesRoot() const manualsDir = getManualsFilesRoot()
const fullPath = join(manualsDir, filePath) const fullPath = join(manualsDir, filePath)
const normalizedFullPath = fullPath.replace(/\\/g, "/")
// Normalize paths to handle both forward and backward slashes const normalizedManualsDir = manualsDir.replace(/\\/g, "/")
const normalizedFullPath = fullPath.replace(/\\/g, '/')
const normalizedManualsDir = manualsDir.replace(/\\/g, '/') if (!existsSync(normalizedFullPath) && !existsSync(fullPath)) {
return new NextResponse("File not found", { status: 404 })
// Verify file exists and is within manuals directory
let fileToRead = normalizedFullPath
if (!existsSync(normalizedFullPath)) {
// Try with original path in case of encoding issues
if (existsSync(fullPath)) {
fileToRead = fullPath
} else {
// Try with different path separators
const altPath = fullPath.replace(/\//g, '\\')
if (existsSync(altPath)) {
fileToRead = altPath
} else {
console.error(`File not found: ${normalizedFullPath}`)
console.error(`Also tried: ${fullPath}`)
return new NextResponse('File not found', { status: 404 })
}
}
} }
// Verify the file is within the manuals directory (security check) const fileToRead = existsSync(normalizedFullPath) ? normalizedFullPath : fullPath
const resolvedPath = fileToRead.replace(/\\/g, '/') const resolvedPath = fileToRead.replace(/\\/g, "/")
if (!resolvedPath.startsWith(normalizedManualsDir.replace(/\\/g, '/'))) { if (!resolvedPath.startsWith(normalizedManualsDir)) {
console.error(`Security violation: Path outside manuals directory: ${resolvedPath}`) return new NextResponse("Invalid path", { status: 400 })
return new NextResponse('Invalid path', { status: 400 })
} }
// Only allow PDF files
if (!filePath.toLowerCase().endsWith('.pdf')) {
return new NextResponse('Invalid file type', { status: 400 })
}
// Read and serve the file
const fileBuffer = await readFile(fileToRead) const fileBuffer = await readFile(fileToRead)
// Get filename for Content-Disposition header
const filename = decodedPath[decodedPath.length - 1]
return new NextResponse(fileBuffer, { return new NextResponse(fileBuffer, {
headers: { headers: {
'Content-Type': 'application/pdf', "Content-Type": "application/pdf",
'Content-Disposition': `inline; filename="${encodeURIComponent(filename)}"`, "Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "manual.pdf")}"`,
'Cache-Control': 'public, max-age=31536000, immutable', "Cache-Control": "public, max-age=31536000, immutable",
'X-Content-Type-Options': 'nosniff', "X-Content-Type-Options": "nosniff",
}, },
}) })
} catch (error) { } catch (error) {
console.error('Error serving manual:', error) console.error("[manuals-api] failed to serve manual", error)
if (error instanceof Error) { return new NextResponse("Internal server error", { status: 500 })
console.error('Error details:', error.message, error.stack)
}
return new NextResponse('Internal server error', { status: 500 })
} }
} }

171
app/api/thumbnails/[...path]/route.ts
View file

@ -1,118 +1,97 @@
import { NextRequest, NextResponse } from 'next/server' import { NextResponse } from "next/server"
import { readFile } from 'fs/promises' import { readFile } from "node:fs/promises"
import { existsSync } from 'fs' import { existsSync } from "node:fs"
import { join } from 'path' import { join } from "node:path"
import { getManualsThumbnailsRoot } from '@/lib/manuals-paths' import { getManualsThumbnailsRoot } from "@/lib/manuals-paths"
import { getManualAssetFromStorage } from "@/lib/manuals-object-storage"
// API routes are not supported in static export (GHL hosting) export const dynamic = "force-dynamic"
// Thumbnails are now served as static files from /thumbnails/
export const dynamic = 'force-static'
/** function decodeSegments(pathArray: string[]) {
* API route to serve thumbnail images for PDF manuals return pathArray.map((segment) => {
* This allows serving files from outside the public folder try {
* let decoded = segment
* Usage: /api/thumbnails/BevMax/manual.jpg while (decoded !== decodeURIComponent(decoded)) {
* NOTE: This route is disabled for static export. Use /thumbnails/ paths instead. decoded = decodeURIComponent(decoded)
*/ }
return decoded
} catch {
return segment
}
})
}
// Required for static export - returns empty array to skip this route function invalidPath(pathValue: string) {
export async function generateStaticParams(): Promise<Array<{ path: string[] }>> { return pathValue.includes("..") || pathValue.startsWith("/")
return [] }
function getContentType(pathValue: string) {
if (pathValue.toLowerCase().endsWith(".png")) {
return "image/png"
}
if (pathValue.toLowerCase().endsWith(".webp")) {
return "image/webp"
}
return "image/jpeg"
} }
export async function GET( export async function GET(
request: NextRequest, _request: Request,
{ params }: { params: Promise<{ path: string[] }> } { params }: { params: Promise<{ path: string[] }> },
) { ) {
try { try {
const { path: pathArray } = await params const { path: pathArray } = await params
const decodedPath = decodeSegments(pathArray)
// Decode URL-encoded path segments - handle multiple encodings const filePath = decodedPath.join("/")
const decodedPath = pathArray.map(segment => { const lowerPath = filePath.toLowerCase()
try {
// Try decoding multiple times in case of double encoding if (
let decoded = segment invalidPath(filePath) ||
while (decoded !== decodeURIComponent(decoded)) { (!lowerPath.endsWith(".jpg") && !lowerPath.endsWith(".jpeg") && !lowerPath.endsWith(".png") && !lowerPath.endsWith(".webp"))
decoded = decodeURIComponent(decoded) ) {
} return new NextResponse("Invalid path", { status: 400 })
return decoded
} catch {
// If decoding fails, use original
return segment
}
})
const filePath = decodedPath.join('/')
// Security: Prevent directory traversal
if (filePath.includes('..') || filePath.startsWith('/')) {
return new NextResponse('Invalid path', { status: 400 })
} }
// Construct full path to thumbnail const storageObject = await getManualAssetFromStorage("thumbnails", filePath)
if (storageObject) {
return new NextResponse(Buffer.from(storageObject.body), {
headers: {
"Content-Type": storageObject.contentType || getContentType(filePath),
"Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "thumbnail.jpg")}"`,
"Cache-Control": "public, max-age=31536000, immutable",
"X-Content-Type-Options": "nosniff",
},
})
}
const thumbnailsDir = getManualsThumbnailsRoot() const thumbnailsDir = getManualsThumbnailsRoot()
const fullPath = join(thumbnailsDir, filePath) const fullPath = join(thumbnailsDir, filePath)
const normalizedFullPath = fullPath.replace(/\\/g, "/")
// Normalize paths to handle both forward and backward slashes const normalizedThumbnailsDir = thumbnailsDir.replace(/\\/g, "/")
const normalizedFullPath = fullPath.replace(/\\/g, '/')
const normalizedThumbnailsDir = thumbnailsDir.replace(/\\/g, '/') if (!existsSync(normalizedFullPath) && !existsSync(fullPath)) {
return new NextResponse("Thumbnail not found", { status: 404 })
// Verify file exists and is within thumbnails directory
let fileToRead = normalizedFullPath
if (!existsSync(normalizedFullPath)) {
// Try with original path in case of encoding issues
if (existsSync(fullPath)) {
fileToRead = fullPath
} else {
// Try with different path separators
const altPath = fullPath.replace(/\//g, '\\')
if (existsSync(altPath)) {
fileToRead = altPath
} else {
// Thumbnail doesn't exist - return 404 (this is expected for manuals without thumbnails)
return new NextResponse('Thumbnail not found', { status: 404 })
}
}
} }
// Verify the file is within the thumbnails directory (security check) const fileToRead = existsSync(normalizedFullPath) ? normalizedFullPath : fullPath
const resolvedPath = fileToRead.replace(/\\/g, '/') const resolvedPath = fileToRead.replace(/\\/g, "/")
if (!resolvedPath.startsWith(normalizedThumbnailsDir.replace(/\\/g, '/'))) { if (!resolvedPath.startsWith(normalizedThumbnailsDir)) {
console.error(`Security violation: Path outside thumbnails directory: ${resolvedPath}`) return new NextResponse("Invalid path", { status: 400 })
return new NextResponse('Invalid path', { status: 400 })
} }
// Only allow image files (jpg, jpeg, png)
const lowerPath = filePath.toLowerCase()
if (!lowerPath.endsWith('.jpg') && !lowerPath.endsWith('.jpeg') && !lowerPath.endsWith('.png')) {
return new NextResponse('Invalid file type', { status: 400 })
}
// Read and serve the file
const fileBuffer = await readFile(fileToRead) const fileBuffer = await readFile(fileToRead)
// Get filename for Content-Disposition header
const filename = decodedPath[decodedPath.length - 1]
// Determine content type
const contentType = lowerPath.endsWith('.png')
? 'image/png'
: 'image/jpeg'
return new NextResponse(fileBuffer, { return new NextResponse(fileBuffer, {
headers: { headers: {
'Content-Type': contentType, "Content-Type": getContentType(filePath),
'Content-Disposition': `inline; filename="${encodeURIComponent(filename)}"`, "Content-Disposition": `inline; filename="${encodeURIComponent(decodedPath.at(-1) || "thumbnail.jpg")}"`,
'Cache-Control': 'public, max-age=31536000, immutable', "Cache-Control": "public, max-age=31536000, immutable",
'X-Content-Type-Options': 'nosniff', "X-Content-Type-Options": "nosniff",
}, },
}) })
} catch (error) { } catch (error) {
console.error('Error serving thumbnail:', error) console.error("[manuals-api] failed to serve thumbnail", error)
if (error instanceof Error) { return new NextResponse("Internal server error", { status: 500 })
console.error('Error details:', error.message, error.stack)
}
return new NextResponse('Internal server error', { status: 500 })
} }
} }

9
app/manuals/dashboard/page.tsx
View file

@ -2,12 +2,20 @@ import { Metadata } from 'next'
import { ManualsDashboardClient } from '@/components/manuals-dashboard-client' import { ManualsDashboardClient } from '@/components/manuals-dashboard-client'
import { PublicInset, PublicPageHeader } from '@/components/public-surface' import { PublicInset, PublicPageHeader } from '@/components/public-surface'
import { getConvexManualDashboard } from '@/lib/convex-service' import { getConvexManualDashboard } from '@/lib/convex-service'
import { businessConfig } from '@/lib/seo-config'
import { readFileSync } from 'fs' import { readFileSync } from 'fs'
import { join } from 'path' import { join } from 'path'
export const metadata: Metadata = { export const metadata: Metadata = {
title: 'Manual Processing Dashboard | Rocky Mountain Vending', title: 'Manual Processing Dashboard | Rocky Mountain Vending',
description: 'Comprehensive dashboard showing processed manual data, statistics, and analytics', description: 'Comprehensive dashboard showing processed manual data, statistics, and analytics',
alternates: {
canonical: `${businessConfig.website}/manuals/dashboard`,
},
robots: {
index: false,
follow: false,
},
} }
interface DashboardData { interface DashboardData {
@ -99,4 +107,3 @@ export default async function ManualsDashboardPage() {

34
app/manuals/page.tsx
View file

@ -10,17 +10,8 @@ import {
groupManuals, groupManuals,
getManufacturers, getManufacturers,
getCategories, getCategories,
filterMDBManuals,
filterManualsBySite,
filterManufacturersByMinCount,
} from '@/lib/manuals' } from '@/lib/manuals'
import { import { selectManualsForSite } from '@/lib/manuals-site-selection'
getSiteDomain,
getAllowedManufacturers,
getMinManualCount,
getManufacturerAliases,
shouldIncludePaymentComponents,
} from '@/lib/site-config'
import { generateStructuredData } from '@/lib/seo' import { generateStructuredData } from '@/lib/seo'
import { getManualsThumbnailsRoot } from '@/lib/manuals-paths' import { getManualsThumbnailsRoot } from '@/lib/manuals-paths'
@ -84,31 +75,10 @@ export const metadata: Metadata = {
} }
export default async function ManualsPage() { export default async function ManualsPage() {
// Get site configuration
const siteDomain = getSiteDomain()
const allowedManufacturers = getAllowedManufacturers(siteDomain)
const minManualCount = getMinManualCount(siteDomain)
const manufacturerAliases = getManufacturerAliases()
const includePaymentComponents = shouldIncludePaymentComponents(siteDomain)
// Prefer Convex-backed metadata, but keep filesystem fallback in place until the shared catalog is fully populated. // Prefer Convex-backed metadata, but keep filesystem fallback in place until the shared catalog is fully populated.
const convexManuals = await listConvexManuals() const convexManuals = await listConvexManuals()
const allManuals = convexManuals.length > 0 ? convexManuals : await scanManuals() const allManuals = convexManuals.length > 0 ? convexManuals : await scanManuals()
let manuals = selectManualsForSite(allManuals).manuals
// Filtering chain:
// 1. Filter by MDB (existing)
let manuals = filterMDBManuals(allManuals)
// 2. Filter by site tier/allowed manufacturers (new)
manuals = filterManualsBySite(
manuals,
allowedManufacturers,
manufacturerAliases,
includePaymentComponents
)
// 3. Filter by minimum manual count per manufacturer (new)
manuals = filterManufacturersByMinCount(manuals, minManualCount)
// Hide broken local thumbnails so the public manuals page doesn't spam 404s. // Hide broken local thumbnails so the public manuals page doesn't spam 404s.
const thumbnailsRoot = getManualsThumbnailsRoot() const thumbnailsRoot = getManualsThumbnailsRoot()

221
lib/manuals-object-storage.ts Normal file
View file

@ -0,0 +1,221 @@
import { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
type AssetKind = "manuals" | "thumbnails"
type StorageConfig = {
endpoint: string
region: string
accessKeyId: string
secretAccessKey: string
forcePathStyle: boolean
disableSse: boolean
disableChecksums: boolean
manualsBucket: string
thumbnailsBucket: string
}
type StoredObject = {
body: Uint8Array
contentType?: string
contentLength?: number
lastModified?: Date
etag?: string
}
let cachedClient: S3Client | null = null
let cachedSignature: string | null = null
function readOptionalEnv(...keys: string[]) {
for (const key of keys) {
const value = String(process.env[key] ?? "").trim()
if (value) {
return value
}
}
return ""
}
function readBooleanEnv(keys: string[], fallback: boolean) {
for (const key of keys) {
const value = String(process.env[key] ?? "").trim().toLowerCase()
if (!value) {
continue
}
if (["1", "true", "yes", "on"].includes(value)) {
return true
}
if (["0", "false", "no", "off"].includes(value)) {
return false
}
}
return fallback
}
function getStorageConfig(): StorageConfig | null {
const endpoint = readOptionalEnv("MANUALS_STORAGE_ENDPOINT", "S3_ENDPOINT_URL", "CLOUDFLARE_R2_ENDPOINT")
const accessKeyId = readOptionalEnv(
"MANUALS_STORAGE_ACCESS_KEY_ID",
"CLOUDFLARE_R2_ACCESS_KEY_ID",
"AWS_ACCESS_KEY_ID",
"AWS_ACCESS_KEY",
)
const secretAccessKey = readOptionalEnv(
"MANUALS_STORAGE_SECRET_ACCESS_KEY",
"CLOUDFLARE_R2_SECRET_ACCESS_KEY",
"AWS_SECRET_ACCESS_KEY",
"AWS_SECRET_KEY",
)
const region = readOptionalEnv("MANUALS_STORAGE_REGION", "AWS_REGION") || "us-east-1"
const manualsBucket = readOptionalEnv("MANUALS_STORAGE_MANUALS_BUCKET", "R2_MANUALS_BUCKET")
const thumbnailsBucket = readOptionalEnv("MANUALS_STORAGE_THUMBNAILS_BUCKET", "R2_THUMBNAILS_BUCKET")
if (!endpoint || !accessKeyId || !secretAccessKey || !manualsBucket || !thumbnailsBucket) {
return null
}
return {
endpoint,
region,
accessKeyId,
secretAccessKey,
forcePathStyle: readBooleanEnv(["MANUALS_STORAGE_FORCE_PATH_STYLE", "AWS_S3_FORCE_PATH_STYLE"], true),
disableSse: readBooleanEnv(["MANUALS_STORAGE_DISABLE_SSE", "AWS_S3_DISABLE_SSE"], true),
disableChecksums: readBooleanEnv(
["MANUALS_STORAGE_DISABLE_CHECKSUMS", "AWS_S3_DISABLE_CHECKSUMS"],
true,
),
manualsBucket,
thumbnailsBucket,
}
}
function getBucketName(kind: AssetKind, config: StorageConfig) {
return kind === "manuals" ? config.manualsBucket : config.thumbnailsBucket
}
function getClient(config: StorageConfig) {
const signature = JSON.stringify(config)
if (!cachedClient || cachedSignature !== signature) {
cachedClient = new S3Client({
region: config.region,
endpoint: config.endpoint,
forcePathStyle: config.forcePathStyle,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
})
cachedSignature = signature
}
return cachedClient
}
function guessContentType(key: string) {
const lower = key.toLowerCase()
if (lower.endsWith(".pdf")) {
return "application/pdf"
}
if (lower.endsWith(".png")) {
return "image/png"
}
if (lower.endsWith(".webp")) {
return "image/webp"
}
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) {
return "image/jpeg"
}
return "application/octet-stream"
}
export function hasManualObjectStorageConfig() {
return getStorageConfig() !== null
}
export async function getManualAssetFromStorage(kind: AssetKind, key: string): Promise<StoredObject | null> {
const config = getStorageConfig()
if (!config) {
return null
}
try {
const response = await getClient(config).send(
new GetObjectCommand({
Bucket: getBucketName(kind, config),
Key: key,
}),
)
if (!response.Body) {
return null
}
return {
body: await response.Body.transformToByteArray(),
contentType: response.ContentType,
contentLength: response.ContentLength,
lastModified: response.LastModified,
etag: response.ETag,
}
} catch (error) {
const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
if (statusCode === 404 || statusCode === 403) {
return null
}
throw error
}
}
export async function manualAssetExists(kind: AssetKind, key: string) {
const config = getStorageConfig()
if (!config) {
return false
}
try {
await getClient(config).send(
new HeadObjectCommand({
Bucket: getBucketName(kind, config),
Key: key,
}),
)
return true
} catch (error) {
const statusCode = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
if (statusCode === 404) {
return false
}
throw error
}
}
export async function uploadManualAsset(
kind: AssetKind,
key: string,
body: Uint8Array,
contentType?: string,
) {
const config = getStorageConfig()
if (!config) {
throw new Error("Manual object storage is not configured.")
}
await getClient(config).send(
new PutObjectCommand({
Bucket: getBucketName(kind, config),
Key: key,
Body: body,
ContentType: contentType || guessContentType(key),
}),
)
}

50
lib/manuals-site-selection.ts Normal file
View file

@ -0,0 +1,50 @@
import type { Manual } from "@/lib/manuals-types"
import {
filterMDBManuals,
filterManualsBySite,
filterManufacturersByMinCount,
} from "@/lib/manuals"
import {
getAllowedManufacturers,
getManufacturerAliases,
getMinManualCount,
getSiteDomain,
shouldIncludePaymentComponents,
} from "@/lib/site-config"
export type ManualSelectionResult = {
siteDomain: string
totalInput: number
mdbEligible: number
siteVisible: number
finalCount: number
manuals: Manual[]
}
export function selectManualsForSite(
manuals: Manual[],
siteDomain: string = getSiteDomain(),
): ManualSelectionResult {
const allowedManufacturers = getAllowedManufacturers(siteDomain)
const manufacturerAliases = getManufacturerAliases()
const minManualCount = getMinManualCount(siteDomain)
const includePaymentComponents = shouldIncludePaymentComponents(siteDomain)
const mdbEligible = filterMDBManuals(manuals)
const siteVisible = filterManualsBySite(
mdbEligible,
allowedManufacturers,
manufacturerAliases,
includePaymentComponents,
)
const finalManuals = filterManufacturersByMinCount(siteVisible, minManualCount)
return {
siteDomain,
totalInput: manuals.length,
mdbEligible: mdbEligible.length,
siteVisible: siteVisible.length,
finalCount: finalManuals.length,
manuals: finalManuals,
}
}

6
lib/manuals-storage.ts
View file

@ -13,17 +13,17 @@ function getSiteBaseUrl() {
} }
export function getManualsAssetBaseUrl() { export function getManualsAssetBaseUrl() {
return cleanBaseUrl(process.env.NEXT_PUBLIC_MANUALS_BASE_URL || `${getSiteBaseUrl()}/manuals`) return cleanBaseUrl(process.env.NEXT_PUBLIC_MANUALS_BASE_URL || `${getSiteBaseUrl()}/api/manuals`)
} }
export function getThumbnailsAssetBaseUrl() { export function getThumbnailsAssetBaseUrl() {
return cleanBaseUrl(process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL || `${getSiteBaseUrl()}/thumbnails`) return cleanBaseUrl(process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL || `${getSiteBaseUrl()}/api/thumbnails`)
} }
export function getManualsAssetSource() { export function getManualsAssetSource() {
return process.env.NEXT_PUBLIC_MANUALS_BASE_URL && process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL return process.env.NEXT_PUBLIC_MANUALS_BASE_URL && process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL
? "external-object-storage" ? "external-object-storage"
: "site-static" : "site-proxy"
} }
export function buildManualAssetUrl(relativePath: string) { export function buildManualAssetUrl(relativePath: string) {

67
lib/manuals-types.ts
View file

@ -1,3 +1,5 @@
import { buildManualAssetUrl, buildThumbnailAssetUrl } from "@/lib/manuals-storage"
export interface Manual { export interface Manual {
filename: string filename: string
path: string path: string
@ -46,40 +48,7 @@ export function getManualUrl(manual: Manual): string {
relativePath = manual.path relativePath = manual.path
} }
// Encode each path segment to handle special characters (spaces, %, etc.) return buildManualAssetUrl(relativePath)
// Split by '/' and encode each segment individually, then rejoin with '/'
// This preserves the path structure while encoding special characters
const pathSegments = relativePath.split('/')
const encodedSegments = pathSegments.map(segment => {
// Only encode if the segment contains characters that need encoding
// This avoids double-encoding already encoded segments
try {
const decoded = decodeURIComponent(segment)
// If decoding succeeds and is different, it was already encoded
if (decoded !== segment) {
return segment // Keep as-is if already encoded
}
} catch {
// If decoding fails, it's not encoded, so encode it
}
return encodeURIComponent(segment)
})
const encodedPath = encodedSegments.join('/')
// Check for external base URL (e.g., Cloudflare)
const baseUrl = process.env.NEXT_PUBLIC_MANUALS_BASE_URL
if (baseUrl) {
// Remove trailing slash from base URL if present
const cleanBaseUrl = baseUrl.replace(/\/$/, '')
return `${cleanBaseUrl}/${encodedPath}`
}
if (process.env.NODE_ENV === 'development') {
return `/api/manuals/${encodedPath}`
}
// Use local static path for exported hosting
return `/manuals/${encodedPath}`
} }
/** /**
@ -109,33 +78,5 @@ export function getThumbnailUrl(manual: Manual): string | null {
relativePath = manual.thumbnailUrl relativePath = manual.thumbnailUrl
} }
// Encode each path segment to handle special characters return buildThumbnailAssetUrl(relativePath)
const pathSegments = relativePath.split('/')
const encodedSegments = pathSegments.map(segment => {
try {
const decoded = decodeURIComponent(segment)
if (decoded !== segment) {
return segment // Keep as-is if already encoded
}
} catch {
// If decoding fails, it's not encoded, so encode it
}
return encodeURIComponent(segment)
})
const encodedPath = encodedSegments.join('/')
// Check for external base URL (e.g., Cloudflare)
const baseUrl = process.env.NEXT_PUBLIC_THUMBNAILS_BASE_URL
if (baseUrl) {
// Remove trailing slash from base URL if present
const cleanBaseUrl = baseUrl.replace(/\/$/, '')
return `${cleanBaseUrl}/${encodedPath}`
}
if (process.env.NODE_ENV === 'development') {
return `/api/thumbnails/${encodedPath}`
}
// Use local static path for exported hosting
return `/thumbnails/${encodedPath}`
} }

16
lib/site-manufacturer-mapping.json
View file

@ -15,6 +15,21 @@
"includePaymentComponents": true, "includePaymentComponents": true,
"minManualCount": 3 "minManualCount": 3
}, },
"rmv.abundancepartners.app": {
"tier": 1,
"description": "Rocky Mountain Vending staging host mirrors the production Rocky manuals subset",
"manufacturers": [
"Crane",
"Royal Vendors",
"GPL",
"AP",
"Dixie-Narco",
"USI",
"Vendo"
],
"includePaymentComponents": true,
"minManualCount": 3
},
"vending.support": { "vending.support": {
"tier": 2, "tier": 2,
"description": "Site with medium manufacturer count - includes Tier 1 plus additional popular manufacturers", "description": "Site with medium manufacturer count - includes Tier 1 plus additional popular manufacturers",
@ -73,4 +88,3 @@
} }

12
scripts/deploy-readiness.mjs
View file

@ -15,7 +15,7 @@ const REQUIRED_ENV_GROUPS = [
}, },
{ {
label: "Manual asset delivery", label: "Manual asset delivery",
keys: ["NEXT_PUBLIC_MANUALS_BASE_URL", "NEXT_PUBLIC_THUMBNAILS_BASE_URL", "CLOUDFLARE_R2_ENDPOINT", "R2_MANUALS_BUCKET", "R2_THUMBNAILS_BUCKET"], keys: ["R2_MANUALS_BUCKET", "R2_THUMBNAILS_BUCKET"],
}, },
{ {
label: "Admin and auth", label: "Admin and auth",
@ -109,6 +109,14 @@ function hasManualStorageCredentials() {
].every(Boolean) ].every(Boolean)
} }
function hasManualStorageEndpoint() {
return Boolean(
readValue("MANUALS_STORAGE_ENDPOINT") ||
readValue("S3_ENDPOINT_URL") ||
readValue("CLOUDFLARE_R2_ENDPOINT"),
)
}
function heading(title) { function heading(title) {
console.log(`\n== ${title} ==`) console.log(`\n== ${title} ==`)
} }
@ -189,6 +197,8 @@ function main() {
if (!hasManualStorageCredentials()) { if (!hasManualStorageCredentials()) {
failures.push("Manual asset storage credentials are incomplete. Set R2/S3 access key and secret env vars before release.") failures.push("Manual asset storage credentials are incomplete. Set R2/S3 access key and secret env vars before release.")
} else if (!hasManualStorageEndpoint()) {
failures.push("Manual asset storage endpoint is incomplete. Set S3_ENDPOINT_URL or CLOUDFLARE_R2_ENDPOINT before release.")
} else { } else {
console.log("Manual asset storage credentials: present") console.log("Manual asset storage credentials: present")
} }

122
scripts/sync-manuals-to-convex.ts
View file

@ -1,11 +1,19 @@
import { spawnSync } from "node:child_process" import { readFile } from "node:fs/promises"
import { join } from "node:path"
import { config as loadEnv } from "dotenv" import { config as loadEnv } from "dotenv"
import { ConvexHttpClient } from "convex/browser" import { ConvexHttpClient } from "convex/browser"
import { makeFunctionReference } from "convex/server" import { makeFunctionReference } from "convex/server"
import { scanManuals } from "../lib/manuals" import { scanManuals } from "../lib/manuals"
import { buildManualAssetUrl, buildThumbnailAssetUrl, getManualsAssetSource } from "../lib/manuals-storage" import { getManualsAssetSource, buildManualAssetUrl, buildThumbnailAssetUrl } from "../lib/manuals-storage"
import { businessConfig } from "../lib/seo-config" import { businessConfig } from "../lib/seo-config"
import { getSiteDomain } from "../lib/site-config" import { getSiteDomain } from "../lib/site-config"
import { selectManualsForSite } from "../lib/manuals-site-selection"
import { getManualsFilesRoot, getManualsThumbnailsRoot } from "../lib/manuals-paths"
import {
hasManualObjectStorageConfig,
manualAssetExists,
uploadManualAsset,
} from "../lib/manuals-object-storage"
loadEnv({ path: ".env.local" }) loadEnv({ path: ".env.local" })
loadEnv({ path: ".env.staging", override: false }) loadEnv({ path: ".env.staging", override: false })
@ -52,35 +60,64 @@ function readConvexUrl() {
return convexUrl return convexUrl
} }
function shouldUploadAssets() { function canonicalSiteVisibility(siteDomain: string) {
const accessKey = const canonicalHost = new URL(businessConfig.website).hostname
process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || return Array.from(new Set([siteDomain, canonicalHost]))
process.env.AWS_ACCESS_KEY_ID ||
process.env.AWS_ACCESS_KEY
const secret =
process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ||
process.env.AWS_SECRET_ACCESS_KEY ||
process.env.AWS_SECRET_KEY
return Boolean(accessKey && secret)
} }
function runAssetUpload() { async function uploadSelectedAssets(
const result = spawnSync("node", ["scripts/upload-to-r2.js", "--type", "all", "--incremental"], { manuals: Awaited<ReturnType<typeof scanManuals>>,
cwd: process.cwd(), dryRun: boolean,
stdio: "inherit", ) {
}) const manualsRoot = getManualsFilesRoot()
const thumbnailsRoot = getManualsThumbnailsRoot()
if ((result.status ?? 1) !== 0) { const stats = {
throw new Error("R2 upload step failed.") manualsUploaded: 0,
manualsSkipped: 0,
thumbnailsUploaded: 0,
thumbnailsSkipped: 0,
} }
for (const manual of manuals) {
const manualKey = manual.path
const manualPath = join(manualsRoot, manual.path)
const shouldSkipManual = await manualAssetExists("manuals", manualKey)
if (shouldSkipManual) {
stats.manualsSkipped += 1
} else if (dryRun) {
stats.manualsUploaded += 1
} else {
await uploadManualAsset("manuals", manualKey, await readFile(manualPath))
stats.manualsUploaded += 1
}
if (!manual.thumbnailUrl) {
continue
}
const thumbnailKey = manual.thumbnailUrl
const thumbnailPath = join(thumbnailsRoot, manual.thumbnailUrl)
const shouldSkipThumbnail = await manualAssetExists("thumbnails", thumbnailKey)
if (shouldSkipThumbnail) {
stats.thumbnailsSkipped += 1
} else if (dryRun) {
stats.thumbnailsUploaded += 1
} else {
await uploadManualAsset("thumbnails", thumbnailKey, await readFile(thumbnailPath))
stats.thumbnailsUploaded += 1
}
}
return stats
} }
function normalizeManualForConvex( function normalizeManualForConvex(
manual: Awaited<ReturnType<typeof scanManuals>>[number], manual: Awaited<ReturnType<typeof scanManuals>>[number],
importBatch: string, importBatch: string,
siteDomain: string,
) { ) {
const siteDomain = getSiteDomain()
const manualUrl = buildManualAssetUrl(manual.path) const manualUrl = buildManualAssetUrl(manual.path)
const thumbnailUrl = manual.thumbnailUrl ? buildThumbnailAssetUrl(manual.thumbnailUrl) : undefined const thumbnailUrl = manual.thumbnailUrl ? buildThumbnailAssetUrl(manual.thumbnailUrl) : undefined
@ -100,32 +137,42 @@ function normalizeManualForConvex(
sourcePath: manual.path, sourcePath: manual.path,
sourceSite: businessConfig.name, sourceSite: businessConfig.name,
sourceDomain: siteDomain, sourceDomain: siteDomain,
siteVisibility: [siteDomain], siteVisibility: canonicalSiteVisibility(siteDomain),
importBatch, importBatch,
} satisfies ManualUpsertInput } satisfies ManualUpsertInput
} }
async function main() { async function main() {
const args = parseArgs(process.argv.slice(2)) const args = parseArgs(process.argv.slice(2))
const siteDomain = getSiteDomain()
if (!args.skipUpload && shouldUploadAssets()) { const allManuals = await scanManuals()
console.log("[manuals-sync] uploading new manuals and thumbnails to object storage") const selection = selectManualsForSite(allManuals, siteDomain)
runAssetUpload() const selectedManuals = args.limit ? selection.manuals.slice(0, args.limit) : selection.manuals
} else if (!args.skipUpload) { const manualsWithThumbnails = selectedManuals.filter((manual) => Boolean(manual.thumbnailUrl))
console.log("[manuals-sync] skipping asset upload because Cloudflare R2 credentials are not configured")
}
const manuals = await scanManuals()
const selectedManuals = args.limit ? manuals.slice(0, args.limit) : manuals
const importBatch = new Date().toISOString() const importBatch = new Date().toISOString()
const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual, importBatch)) const payload = selectedManuals.map((manual) => normalizeManualForConvex(manual, importBatch, siteDomain))
console.log("[manuals-sync] scanned manuals", { console.log("[manuals-sync] selection", {
total: manuals.length, siteDomain: selection.siteDomain,
selected: selectedManuals.length, scanned: selection.totalInput,
mdbEligible: selection.mdbEligible,
siteVisible: selection.siteVisible,
finalSelected: selection.finalCount,
selectedForRun: selectedManuals.length,
thumbnailsForRun: manualsWithThumbnails.length,
assetSource: getManualsAssetSource(), assetSource: getManualsAssetSource(),
}) })
if (!args.skipUpload) {
if (!hasManualObjectStorageConfig()) {
throw new Error("Manual object storage credentials are not configured.")
}
console.log("[manuals-sync] uploading selected manuals and thumbnails")
const uploadStats = await uploadSelectedAssets(selectedManuals, args.dryRun)
console.log("[manuals-sync] upload summary", uploadStats)
}
if (args.dryRun) { if (args.dryRun) {
console.log("[manuals-sync] dry run complete") console.log("[manuals-sync] dry run complete")
return return
@ -148,6 +195,7 @@ async function main() {
console.log("[manuals-sync] completed", { console.log("[manuals-sync] completed", {
synced, synced,
importBatch, importBatch,
siteDomain,
}) })
} }

41
scripts/upload-to-r2.js
View file

@ -1,7 +1,7 @@
#!/usr/bin/env node #!/usr/bin/env node
/** /**
* Upload scripts to Cloudflare R2 using S3-compatible API * Upload manuals assets to any S3-compatible object storage (Cloudflare R2, MinIO, etc.)
* *
* Usage: * Usage:
* node scripts/upload-to-r2.js --type manuals --dry-run * node scripts/upload-to-r2.js --type manuals --dry-run
@ -9,10 +9,8 @@
* node scripts/upload-to-r2.js --type all * node scripts/upload-to-r2.js --type all
* *
* Environment variables required: * Environment variables required:
* CLOUDFLARE_R2_ACCESS_KEY_ID * S3_ENDPOINT_URL or CLOUDFLARE_R2_ENDPOINT
* CLOUDFLARE_R2_SECRET_ACCESS_KEY * AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* variants)
* CLOUDFLARE_R2_ENDPOINT (or uses default with account ID)
* CLOUDFLARE_ACCOUNT_ID
*/ */
import { S3Client, PutObjectCommand, ListObjectsV2Command, HeadObjectCommand } from '@aws-sdk/client-s3'; import { S3Client, PutObjectCommand, ListObjectsV2Command, HeadObjectCommand } from '@aws-sdk/client-s3';
@ -21,10 +19,25 @@ import { join, relative, dirname } from 'path';
import { existsSync } from 'fs'; import { existsSync } from 'fs';
const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || 'bd6f76304a840ba11b75f9ced84264f4'; const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || 'bd6f76304a840ba11b75f9ced84264f4';
const ENDPOINT = process.env.CLOUDFLARE_R2_ENDPOINT || `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`; const ENDPOINT =
const ACCESS_KEY_ID = process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY; process.env.MANUALS_STORAGE_ENDPOINT ||
process.env.S3_ENDPOINT_URL ||
process.env.CLOUDFLARE_R2_ENDPOINT ||
`https://${ACCOUNT_ID}.r2.cloudflarestorage.com`;
const ACCESS_KEY_ID =
process.env.MANUALS_STORAGE_ACCESS_KEY_ID ||
process.env.CLOUDFLARE_R2_ACCESS_KEY_ID ||
process.env.AWS_ACCESS_KEY_ID ||
process.env.AWS_ACCESS_KEY;
const SECRET_ACCESS_KEY = const SECRET_ACCESS_KEY =
process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY || process.env.AWS_SECRET_KEY; process.env.MANUALS_STORAGE_SECRET_ACCESS_KEY ||
process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ||
process.env.AWS_SECRET_ACCESS_KEY ||
process.env.AWS_SECRET_KEY;
const REGION = process.env.MANUALS_STORAGE_REGION || process.env.AWS_REGION || 'us-east-1';
const FORCE_PATH_STYLE = ['1', 'true', 'yes', 'on'].includes(
String(process.env.MANUALS_STORAGE_FORCE_PATH_STYLE || process.env.AWS_S3_FORCE_PATH_STYLE || 'true').toLowerCase()
);
// Bucket names // Bucket names
const MANUALS_BUCKET = process.env.R2_MANUALS_BUCKET || 'vending-vm-manuals'; const MANUALS_BUCKET = process.env.R2_MANUALS_BUCKET || 'vending-vm-manuals';
@ -46,16 +59,17 @@ const uploadType = typeArg || 'all';
const dryRun = args.includes('--dry-run') || args.includes('-d'); const dryRun = args.includes('--dry-run') || args.includes('-d');
const incremental = args.includes('--incremental') || args.includes('-i'); const incremental = args.includes('--incremental') || args.includes('-i');
if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY) { if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY || !ENDPOINT) {
console.error('❌ Error: CLOUDFLARE_R2_ACCESS_KEY_ID and CLOUDFLARE_R2_SECRET_ACCESS_KEY must be set'); console.error('❌ Error: S3-compatible manuals storage env vars are incomplete');
console.error(' Set these in your .env file or environment variables'); console.error(' Set S3_ENDPOINT_URL plus AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* equivalents)');
process.exit(1); process.exit(1);
} }
// Initialize S3 client // Initialize S3 client
const s3Client = new S3Client({ const s3Client = new S3Client({
region: 'auto', region: REGION,
endpoint: ENDPOINT, endpoint: ENDPOINT,
forcePathStyle: FORCE_PATH_STYLE,
credentials: { credentials: {
accessKeyId: ACCESS_KEY_ID, accessKeyId: ACCESS_KEY_ID,
secretAccessKey: SECRET_ACCESS_KEY, secretAccessKey: SECRET_ACCESS_KEY,
@ -206,8 +220,7 @@ async function uploadDirectory(sourceDir, bucket, bucketName, dryRun = false, in
* Main upload function * Main upload function
*/ */
async function main() { async function main() {
console.log('🚀 Cloudflare R2 Upload Script'); console.log('🚀 Manuals Object Storage Upload Script');
console.log(` Account ID: ${ACCOUNT_ID}`);
console.log(` Endpoint: ${ENDPOINT}`); console.log(` Endpoint: ${ENDPOINT}`);
console.log(` Mode: ${dryRun ? 'DRY RUN' : 'UPLOAD'}`); console.log(` Mode: ${dryRun ? 'DRY RUN' : 'UPLOAD'}`);
console.log(` Incremental: ${incremental ? 'Yes' : 'No'}`); console.log(` Incremental: ${incremental ? 'Yes' : 'No'}`);