/** * Backfill: upload base64 data-URL file values in FormSubmission.data to S3 * and replace with CloudFront URLs. * * Run: * cd backend && npm run backfill:form-submission-files * cd backend && npm run backfill:form-submission-files -- --dry-run * cd backend && npm run backfill:form-submission-files -- --form-id= */ const path = require('path'); require('dotenv').config({ path: path.join(__dirname, '..', '.env') }); const mongoose = require('mongoose'); const Form = require(path.join(__dirname, '..', 'src', 'models', 'Form')); const FormSubmission = require(path.join(__dirname, '..', 'src', 'models', 'FormSubmission')); const { processAllSubmissionDataUrls, hasDataUrlValue, dataMapToObject, } = require(path.join(__dirname, '..', 'src', 'utils', 'formSubmissionFiles')); function parseArgs(argv) { const dryRun = argv.includes('--dry-run'); const formIdArg = argv.find((a) => a.startsWith('--form-id=')); const formId = formIdArg ? formIdArg.split('=')[1]?.trim() : ''; const limitArg = argv.find((a) => a.startsWith('--limit=')); const limit = limitArg ? Math.max(1, parseInt(limitArg.split('=')[1], 10) || 0) : 0; return { dryRun, formId, limit }; } function dataChanged(before, after) { return JSON.stringify(before) !== JSON.stringify(after); } async function run() { const { dryRun, formId, limit } = parseArgs(process.argv.slice(2)); const uri = process.env.MONGODB_URI; if (!uri) { console.error('MONGODB_URI is not set'); process.exit(1); } if (!process.env.AWS_BUCKET || !process.env.AWS_URL) { console.error('AWS_BUCKET and AWS_URL must be set for S3 upload'); process.exit(1); } await mongoose.connect(uri); console.log(`Connected. dryRun=${dryRun}${formId ? ` formId=${formId}` : ''}${limit ? ` limit=${limit}` : ''}`); const query = {}; if (formId) { if (!mongoose.Types.ObjectId.isValid(formId)) { console.error('Invalid --form-id'); process.exit(1); } query.form = formId; } let cursor = FormSubmission.find(query).sort({ submittedAt: 1 }).cursor(); const formCache = new Map(); let scanned = 0; let candidates = 0; let updated = 0; let skipped = 0; let failed = 0; for await (const submission of cursor) { if (limit && scanned >= limit) break; scanned += 1; if (!hasDataUrlValue(submission.data)) { skipped += 1; continue; } candidates += 1; const formKey = String(submission.form); let form = formCache.get(formKey); if (form === undefined) { form = submission.form ? await Form.findById(submission.form).lean() : null; formCache.set(formKey, form || null); } const before = dataMapToObject(submission.data); const orgId = String(submission.tenantOrganization || form?.tenantOrganization || 'org'); const eventId = submission.event ? String(submission.event) : ''; try { const after = await processAllSubmissionDataUrls(before, { orgId, eventId, form }); if (!dataChanged(before, after)) { skipped += 1; continue; } if (dryRun) { const keys = Object.keys(before).filter((k) => before[k] !== after[k]); console.log(`[dry-run] submission ${submission._id} would update keys: ${keys.join(', ')}`); updated += 1; continue; } submission.data = after; await submission.save(); updated += 1; console.log(`Updated submission ${submission._id}`); } catch (err) { failed += 1; console.error(`Failed submission ${submission._id}:`, err.message || err); } } console.log( `Done. scanned=${scanned} candidates=${candidates} updated=${updated} skipped=${skipped} failed=${failed}`, ); await mongoose.disconnect(); } run().catch((err) => { console.error(err); process.exit(1); });