#!/usr/bin/env node /** * Upload manuals assets to any S3-compatible object storage (Cloudflare R2, MinIO, etc.) * * Usage: * node scripts/upload-to-r2.js --type manuals --dry-run * node scripts/upload-to-r2.js --type thumbnails * node scripts/upload-to-r2.js --type all * * Environment variables required: * S3_ENDPOINT_URL or CLOUDFLARE_R2_ENDPOINT * AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* variants) */ import { S3Client, PutObjectCommand, ListObjectsV2Command, HeadObjectCommand, } from "@aws-sdk/client-s3" import { readdir, stat, readFile } from "fs/promises" import { join, relative, dirname } from "path" import { existsSync } from "fs" const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || "bd6f76304a840ba11b75f9ced84264f4" const ENDPOINT = process.env.MANUALS_STORAGE_ENDPOINT || process.env.S3_ENDPOINT_URL || process.env.CLOUDFLARE_R2_ENDPOINT || `https://${ACCOUNT_ID}.r2.cloudflarestorage.com` const ACCESS_KEY_ID = process.env.MANUALS_STORAGE_ACCESS_KEY_ID || process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY const SECRET_ACCESS_KEY = process.env.MANUALS_STORAGE_SECRET_ACCESS_KEY || process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY || process.env.AWS_SECRET_KEY const REGION = process.env.MANUALS_STORAGE_REGION || process.env.AWS_REGION || "us-east-1" const FORCE_PATH_STYLE = ["1", "true", "yes", "on"].includes( String( process.env.MANUALS_STORAGE_FORCE_PATH_STYLE || process.env.AWS_S3_FORCE_PATH_STYLE || "true" ).toLowerCase() ) // Bucket names const MANUALS_BUCKET = process.env.R2_MANUALS_BUCKET || "vending-vm-manuals" const THUMBNAILS_BUCKET = process.env.R2_THUMBNAILS_BUCKET || "vending-vm-thumbnails" // Source directories (relative to project root) const MANUALS_DATA_ROOT = process.env.MANUALS_DATA_ROOT || (existsSync(join(process.cwd(), "..", "manuals-data")) ? join(process.cwd(), "..", "manuals-data") : "/Users/matthewcall/Documents/VS Code Projects/Rocky Mountain Vending/manuals-data") const MANUALS_SOURCE = join(MANUALS_DATA_ROOT, "manuals") const THUMBNAILS_SOURCE = join(MANUALS_DATA_ROOT, "thumbnails") // Parse command line arguments const args = process.argv.slice(2) const typeArg = args.find((arg) => arg.startsWith("--type="))?.split("=")[1] || (args.includes("--type") ? args[args.indexOf("--type") + 1] : null) const uploadType = typeArg || "all" const dryRun = args.includes("--dry-run") || args.includes("-d") const incremental = args.includes("--incremental") || args.includes("-i") if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY || !ENDPOINT) { console.error( "āŒ Error: S3-compatible manuals storage env vars are incomplete" ) console.error( " Set S3_ENDPOINT_URL plus AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* equivalents)" ) process.exit(1) } // Initialize S3 client const s3Client = new S3Client({ region: REGION, endpoint: ENDPOINT, forcePathStyle: FORCE_PATH_STYLE, credentials: { accessKeyId: ACCESS_KEY_ID, secretAccessKey: SECRET_ACCESS_KEY, }, }) /** * Check if object exists in R2 */ async function objectExists(bucket, key) { try { await s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })) return true } catch (error) { if (error.name === "NotFound" || error.$metadata?.httpStatusCode === 404) { return false } throw error } } /** * Get content type based on file extension */ function getContentType(filename) { const ext = filename.split(".").pop()?.toLowerCase() const types = { pdf: "application/pdf", jpg: "image/jpeg", jpeg: "image/jpeg", png: "image/png", webp: "image/webp", gif: "image/gif", } return types[ext] || "application/octet-stream" } /** * Upload a single file to R2 */ async function uploadFile(bucket, key, filePath, dryRun = false) { if (dryRun) { console.log(` [DRY RUN] Would upload: ${key}`) return { uploaded: false, skipped: false, key } } try { const fileContent = await readFile(filePath) const contentType = getContentType(key) await s3Client.send( new PutObjectCommand({ Bucket: bucket, Key: key, Body: fileContent, ContentType: contentType, }) ) return { uploaded: true, skipped: false, key } } catch (error) { console.error(` āŒ Error uploading ${key}:`, error.message) return { uploaded: false, skipped: false, key, error: error.message } } } /** * Recursively get all files from a directory */ async function getAllFiles(dir, baseDir = dir) { const files = [] const entries = await readdir(dir, { withFileTypes: true }) for (const entry of entries) { const fullPath = join(dir, entry.name) if (entry.isDirectory()) { const subFiles = await getAllFiles(fullPath, baseDir) files.push(...subFiles) } else { const relativePath = relative(baseDir, fullPath) files.push({ fullPath, relativePath }) } } return files } /** * Upload directory to R2 bucket */ async function uploadDirectory( sourceDir, bucket, bucketName, dryRun = false, incremental = false ) { if (!existsSync(sourceDir)) { console.error(`āŒ Source directory does not exist: ${sourceDir}`) return { uploaded: 0, skipped: 0, failed: 0, total: 0 } } console.log(`\nšŸ“ Scanning: ${sourceDir}`) const files = await getAllFiles(sourceDir) console.log(` Found ${files.length} files`) if (files.length === 0) { console.log(" No files to upload") return { uploaded: 0, skipped: 0, failed: 0, total: 0 } } let uploaded = 0 let skipped = 0 let failed = 0 console.log( `\n${dryRun ? "šŸ” [DRY RUN]" : "ā¬†ļø "} Uploading to bucket: ${bucketName}` ) for (let i = 0; i < files.length; i++) { const { fullPath, relativePath } = files[i] const key = relativePath.replace(/\\/g, "/") // Normalize path separators // Check if file exists (incremental mode) if (incremental && !dryRun) { const exists = await objectExists(bucket, key) if (exists) { skipped++ if ((i + 1) % 50 === 0) { process.stdout.write( `\r Progress: ${i + 1}/${files.length} (${uploaded} uploaded, ${skipped} skipped, ${failed} failed)` ) } continue } } const result = await uploadFile(bucket, key, fullPath, dryRun) if (result.uploaded) { uploaded++ } else if (result.error) { failed++ } else if (result.skipped) { skipped++ } // Progress indicator if ((i + 1) % 10 === 0 || i + 1 === files.length) { process.stdout.write( `\r Progress: ${i + 1}/${files.length} (${uploaded} uploaded, ${skipped} skipped, ${failed} failed)` ) } } console.log("\n") return { uploaded, skipped, failed, total: files.length } } /** * Main upload function */ async function main() { console.log("šŸš€ Manuals Object Storage Upload Script") console.log(` Endpoint: ${ENDPOINT}`) console.log(` Mode: ${dryRun ? "DRY RUN" : "UPLOAD"}`) console.log(` Incremental: ${incremental ? "Yes" : "No"}`) console.log(` Type: ${uploadType}`) const results = { manuals: { uploaded: 0, skipped: 0, failed: 0, total: 0 }, thumbnails: { uploaded: 0, skipped: 0, failed: 0, total: 0 }, } if (uploadType === "manuals" || uploadType === "all") { console.log("\nšŸ“š Uploading manuals...") results.manuals = await uploadDirectory( MANUALS_SOURCE, MANUALS_BUCKET, MANUALS_BUCKET, dryRun, incremental ) } if (uploadType === "thumbnails" || uploadType === "all") { console.log("\nšŸ–¼ļø Uploading thumbnails...") results.thumbnails = await uploadDirectory( THUMBNAILS_SOURCE, THUMBNAILS_BUCKET, THUMBNAILS_BUCKET, dryRun, incremental ) } // Summary console.log("\nšŸ“Š Upload Summary:") if (uploadType === "manuals" || uploadType === "all") { console.log( ` Manuals: ${results.manuals.uploaded} uploaded, ${results.manuals.skipped} skipped, ${results.manuals.failed} failed (${results.manuals.total} total)` ) } if (uploadType === "thumbnails" || uploadType === "all") { console.log( ` Thumbnails: ${results.thumbnails.uploaded} uploaded, ${results.thumbnails.skipped} skipped, ${results.thumbnails.failed} failed (${results.thumbnails.total} total)` ) } const totalUploaded = results.manuals.uploaded + results.thumbnails.uploaded const totalFailed = results.manuals.failed + results.thumbnails.failed if (totalFailed > 0) { console.log(`\nāš ļø ${totalFailed} file(s) failed to upload`) process.exit(1) } else if (dryRun) { console.log( "\nāœ… Dry run completed. Remove --dry-run to actually upload files." ) } else { console.log(`\nāœ… Successfully uploaded ${totalUploaded} file(s) to R2`) } } main().catch((error) => { console.error("\nāŒ Fatal error:", error) process.exit(1) })