#!/usr/bin/env node /** * Upload manuals assets to any S3-compatible object storage (Cloudflare R2, MinIO, etc.) * * Usage: * node scripts/upload-to-r2.js --type manuals --dry-run * node scripts/upload-to-r2.js --type thumbnails * node scripts/upload-to-r2.js --type all * * Environment variables required: * S3_ENDPOINT_URL or CLOUDFLARE_R2_ENDPOINT * AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* variants) */ import { S3Client, PutObjectCommand, ListObjectsV2Command, HeadObjectCommand } from '@aws-sdk/client-s3'; import { readdir, stat, readFile } from 'fs/promises'; import { join, relative, dirname } from 'path'; import { existsSync } from 'fs'; const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || 'bd6f76304a840ba11b75f9ced84264f4'; const ENDPOINT = process.env.MANUALS_STORAGE_ENDPOINT || process.env.S3_ENDPOINT_URL || process.env.CLOUDFLARE_R2_ENDPOINT || `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`; const ACCESS_KEY_ID = process.env.MANUALS_STORAGE_ACCESS_KEY_ID || process.env.CLOUDFLARE_R2_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY; const SECRET_ACCESS_KEY = process.env.MANUALS_STORAGE_SECRET_ACCESS_KEY || process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY || process.env.AWS_SECRET_KEY; const REGION = process.env.MANUALS_STORAGE_REGION || process.env.AWS_REGION || 'us-east-1'; const FORCE_PATH_STYLE = ['1', 'true', 'yes', 'on'].includes( String(process.env.MANUALS_STORAGE_FORCE_PATH_STYLE || process.env.AWS_S3_FORCE_PATH_STYLE || 'true').toLowerCase() ); // Bucket names const MANUALS_BUCKET = process.env.R2_MANUALS_BUCKET || 'vending-vm-manuals'; const THUMBNAILS_BUCKET = process.env.R2_THUMBNAILS_BUCKET || 'vending-vm-thumbnails'; // Source directories (relative to project root) const MANUALS_DATA_ROOT = process.env.MANUALS_DATA_ROOT || (existsSync(join(process.cwd(), '..', 'manuals-data')) ? join(process.cwd(), '..', 'manuals-data') : '/Users/matthewcall/Documents/VS Code Projects/Rocky Mountain Vending/manuals-data'); const MANUALS_SOURCE = join(MANUALS_DATA_ROOT, 'manuals'); const THUMBNAILS_SOURCE = join(MANUALS_DATA_ROOT, 'thumbnails'); // Parse command line arguments const args = process.argv.slice(2); const typeArg = args.find(arg => arg.startsWith('--type='))?.split('=')[1] || (args.includes('--type') ? args[args.indexOf('--type') + 1] : null); const uploadType = typeArg || 'all'; const dryRun = args.includes('--dry-run') || args.includes('-d'); const incremental = args.includes('--incremental') || args.includes('-i'); if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY || !ENDPOINT) { console.error('āŒ Error: S3-compatible manuals storage env vars are incomplete'); console.error(' Set S3_ENDPOINT_URL plus AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (or CLOUDFLARE_* equivalents)'); process.exit(1); } // Initialize S3 client const s3Client = new S3Client({ region: REGION, endpoint: ENDPOINT, forcePathStyle: FORCE_PATH_STYLE, credentials: { accessKeyId: ACCESS_KEY_ID, secretAccessKey: SECRET_ACCESS_KEY, }, }); /** * Check if object exists in R2 */ async function objectExists(bucket, key) { try { await s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); return true; } catch (error) { if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) { return false; } throw error; } } /** * Get content type based on file extension */ function getContentType(filename) { const ext = filename.split('.').pop()?.toLowerCase(); const types = { 'pdf': 'application/pdf', 'jpg': 'image/jpeg', 'jpeg': 'image/jpeg', 'png': 'image/png', 'webp': 'image/webp', 'gif': 'image/gif', }; return types[ext] || 'application/octet-stream'; } /** * Upload a single file to R2 */ async function uploadFile(bucket, key, filePath, dryRun = false) { if (dryRun) { console.log(` [DRY RUN] Would upload: ${key}`); return { uploaded: false, skipped: false, key }; } try { const fileContent = await readFile(filePath); const contentType = getContentType(key); await s3Client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: fileContent, ContentType: contentType, })); return { uploaded: true, skipped: false, key }; } catch (error) { console.error(` āŒ Error uploading ${key}:`, error.message); return { uploaded: false, skipped: false, key, error: error.message }; } } /** * Recursively get all files from a directory */ async function getAllFiles(dir, baseDir = dir) { const files = []; const entries = await readdir(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { const subFiles = await getAllFiles(fullPath, baseDir); files.push(...subFiles); } else { const relativePath = relative(baseDir, fullPath); files.push({ fullPath, relativePath }); } } return files; } /** * Upload directory to R2 bucket */ async function uploadDirectory(sourceDir, bucket, bucketName, dryRun = false, incremental = false) { if (!existsSync(sourceDir)) { console.error(`āŒ Source directory does not exist: ${sourceDir}`); return { uploaded: 0, skipped: 0, failed: 0, total: 0 }; } console.log(`\nšŸ“ Scanning: ${sourceDir}`); const files = await getAllFiles(sourceDir); console.log(` Found ${files.length} files`); if (files.length === 0) { console.log(' No files to upload'); return { uploaded: 0, skipped: 0, failed: 0, total: 0 }; } let uploaded = 0; let skipped = 0; let failed = 0; console.log(`\n${dryRun ? 'šŸ” [DRY RUN]' : 'ā¬†ļø '} Uploading to bucket: ${bucketName}`); for (let i = 0; i < files.length; i++) { const { fullPath, relativePath } = files[i]; const key = relativePath.replace(/\\/g, '/'); // Normalize path separators // Check if file exists (incremental mode) if (incremental && !dryRun) { const exists = await objectExists(bucket, key); if (exists) { skipped++; if ((i + 1) % 50 === 0) { process.stdout.write(`\r Progress: ${i + 1}/${files.length} (${uploaded} uploaded, ${skipped} skipped, ${failed} failed)`); } continue; } } const result = await uploadFile(bucket, key, fullPath, dryRun); if (result.uploaded) { uploaded++; } else if (result.error) { failed++; } else if (result.skipped) { skipped++; } // Progress indicator if ((i + 1) % 10 === 0 || i + 1 === files.length) { process.stdout.write(`\r Progress: ${i + 1}/${files.length} (${uploaded} uploaded, ${skipped} skipped, ${failed} failed)`); } } console.log('\n'); return { uploaded, skipped, failed, total: files.length }; } /** * Main upload function */ async function main() { console.log('šŸš€ Manuals Object Storage Upload Script'); console.log(` Endpoint: ${ENDPOINT}`); console.log(` Mode: ${dryRun ? 'DRY RUN' : 'UPLOAD'}`); console.log(` Incremental: ${incremental ? 'Yes' : 'No'}`); console.log(` Type: ${uploadType}`); const results = { manuals: { uploaded: 0, skipped: 0, failed: 0, total: 0 }, thumbnails: { uploaded: 0, skipped: 0, failed: 0, total: 0 }, }; if (uploadType === 'manuals' || uploadType === 'all') { console.log('\nšŸ“š Uploading manuals...'); results.manuals = await uploadDirectory( MANUALS_SOURCE, MANUALS_BUCKET, MANUALS_BUCKET, dryRun, incremental ); } if (uploadType === 'thumbnails' || uploadType === 'all') { console.log('\nšŸ–¼ļø Uploading thumbnails...'); results.thumbnails = await uploadDirectory( THUMBNAILS_SOURCE, THUMBNAILS_BUCKET, THUMBNAILS_BUCKET, dryRun, incremental ); } // Summary console.log('\nšŸ“Š Upload Summary:'); if (uploadType === 'manuals' || uploadType === 'all') { console.log(` Manuals: ${results.manuals.uploaded} uploaded, ${results.manuals.skipped} skipped, ${results.manuals.failed} failed (${results.manuals.total} total)`); } if (uploadType === 'thumbnails' || uploadType === 'all') { console.log(` Thumbnails: ${results.thumbnails.uploaded} uploaded, ${results.thumbnails.skipped} skipped, ${results.thumbnails.failed} failed (${results.thumbnails.total} total)`); } const totalUploaded = results.manuals.uploaded + results.thumbnails.uploaded; const totalFailed = results.manuals.failed + results.thumbnails.failed; if (totalFailed > 0) { console.log(`\nāš ļø ${totalFailed} file(s) failed to upload`); process.exit(1); } else if (dryRun) { console.log('\nāœ… Dry run completed. Remove --dry-run to actually upload files.'); } else { console.log(`\nāœ… Successfully uploaded ${totalUploaded} file(s) to R2`); } } main().catch(error => { console.error('\nāŒ Fatal error:', error); process.exit(1); });