126 lines
3.8 KiB
JavaScript
126 lines
3.8 KiB
JavaScript
/**
|
|
* Backfill: upload base64 data-URL file values in FormSubmission.data to S3
|
|
* and replace with CloudFront URLs.
|
|
*
|
|
* Run:
|
|
* cd backend && npm run backfill:form-submission-files
|
|
* cd backend && npm run backfill:form-submission-files -- --dry-run
|
|
* cd backend && npm run backfill:form-submission-files -- --form-id=<mongoId>
|
|
*/
|
|
const path = require('path');
|
|
|
|
require('dotenv').config({ path: path.join(__dirname, '..', '.env') });
|
|
|
|
const mongoose = require('mongoose');
|
|
const Form = require(path.join(__dirname, '..', 'src', 'models', 'Form'));
|
|
const FormSubmission = require(path.join(__dirname, '..', 'src', 'models', 'FormSubmission'));
|
|
const {
|
|
processAllSubmissionDataUrls,
|
|
hasDataUrlValue,
|
|
dataMapToObject,
|
|
} = require(path.join(__dirname, '..', 'src', 'utils', 'formSubmissionFiles'));
|
|
|
|
function parseArgs(argv) {
|
|
const dryRun = argv.includes('--dry-run');
|
|
const formIdArg = argv.find((a) => a.startsWith('--form-id='));
|
|
const formId = formIdArg ? formIdArg.split('=')[1]?.trim() : '';
|
|
const limitArg = argv.find((a) => a.startsWith('--limit='));
|
|
const limit = limitArg ? Math.max(1, parseInt(limitArg.split('=')[1], 10) || 0) : 0;
|
|
return { dryRun, formId, limit };
|
|
}
|
|
|
|
function dataChanged(before, after) {
|
|
return JSON.stringify(before) !== JSON.stringify(after);
|
|
}
|
|
|
|
async function run() {
|
|
const { dryRun, formId, limit } = parseArgs(process.argv.slice(2));
|
|
const uri = process.env.MONGODB_URI;
|
|
if (!uri) {
|
|
console.error('MONGODB_URI is not set');
|
|
process.exit(1);
|
|
}
|
|
|
|
if (!process.env.AWS_BUCKET || !process.env.AWS_URL) {
|
|
console.error('AWS_BUCKET and AWS_URL must be set for S3 upload');
|
|
process.exit(1);
|
|
}
|
|
|
|
await mongoose.connect(uri);
|
|
console.log(`Connected. dryRun=${dryRun}${formId ? ` formId=${formId}` : ''}${limit ? ` limit=${limit}` : ''}`);
|
|
|
|
const query = {};
|
|
if (formId) {
|
|
if (!mongoose.Types.ObjectId.isValid(formId)) {
|
|
console.error('Invalid --form-id');
|
|
process.exit(1);
|
|
}
|
|
query.form = formId;
|
|
}
|
|
|
|
let cursor = FormSubmission.find(query).sort({ submittedAt: 1 }).cursor();
|
|
const formCache = new Map();
|
|
|
|
let scanned = 0;
|
|
let candidates = 0;
|
|
let updated = 0;
|
|
let skipped = 0;
|
|
let failed = 0;
|
|
|
|
for await (const submission of cursor) {
|
|
if (limit && scanned >= limit) break;
|
|
scanned += 1;
|
|
|
|
if (!hasDataUrlValue(submission.data)) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
candidates += 1;
|
|
const formKey = String(submission.form);
|
|
let form = formCache.get(formKey);
|
|
if (form === undefined) {
|
|
form = submission.form ? await Form.findById(submission.form).lean() : null;
|
|
formCache.set(formKey, form || null);
|
|
}
|
|
|
|
const before = dataMapToObject(submission.data);
|
|
const orgId = String(submission.tenantOrganization || form?.tenantOrganization || 'org');
|
|
const eventId = submission.event ? String(submission.event) : '';
|
|
|
|
try {
|
|
const after = await processAllSubmissionDataUrls(before, { orgId, eventId, form });
|
|
if (!dataChanged(before, after)) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
|
|
if (dryRun) {
|
|
const keys = Object.keys(before).filter((k) => before[k] !== after[k]);
|
|
console.log(`[dry-run] submission ${submission._id} would update keys: ${keys.join(', ')}`);
|
|
updated += 1;
|
|
continue;
|
|
}
|
|
|
|
submission.data = after;
|
|
await submission.save();
|
|
updated += 1;
|
|
console.log(`Updated submission ${submission._id}`);
|
|
} catch (err) {
|
|
failed += 1;
|
|
console.error(`Failed submission ${submission._id}:`, err.message || err);
|
|
}
|
|
}
|
|
|
|
console.log(
|
|
`Done. scanned=${scanned} candidates=${candidates} updated=${updated} skipped=${skipped} failed=${failed}`,
|
|
);
|
|
|
|
await mongoose.disconnect();
|
|
}
|
|
|
|
run().catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|