Add event photo search deletion
This commit is contained in:
parent
f74ac6913f
commit
0fe4356b34
18
src/app.js
18
src/app.js
|
|
@ -408,6 +408,24 @@ const app = {
|
||||||
},
|
},
|
||||||
{ timezone: process.env.CRON_TZ || undefined },
|
{ timezone: process.env.CRON_TZ || undefined },
|
||||||
);
|
);
|
||||||
|
// Automatic photo retention cleanup — runs daily at 02:00.
|
||||||
|
const { runPhotoCleanup } = require('./services/photoCleanupService');
|
||||||
|
cron.schedule(
|
||||||
|
'0 2 * * *',
|
||||||
|
async () => {
|
||||||
|
try {
|
||||||
|
const r = await runPhotoCleanup();
|
||||||
|
if (r.deleted > 0 || r.failed > 0) {
|
||||||
|
console.log(
|
||||||
|
`[cron] photo retention cleanup: processed=${r.processed} deleted=${r.deleted} failed=${r.failed} retentionMonths=${r.retentionMonths}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[cron] photo retention cleanup failed', err);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ timezone: process.env.CRON_TZ || undefined },
|
||||||
|
);
|
||||||
const { startEmbeddedQueueWorkers } = require('./utils/queueWorkerKick');
|
const { startEmbeddedQueueWorkers } = require('./utils/queueWorkerKick');
|
||||||
startEmbeddedQueueWorkers();
|
startEmbeddedQueueWorkers();
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
const mongoose = require('mongoose');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* stores `photoRetentionMonths`, which controls how long event photos are kept
|
||||||
|
* before the automatic photo-retention cleanup deletes them.
|
||||||
|
*
|
||||||
|
* A single document (with a fixed, constant `_id`) is used so reads are cheap
|
||||||
|
* and there is never more than one settings row.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const SYSTEM_SETTINGS_ID = new mongoose.Types.ObjectId('000000000000000000000001');
|
||||||
|
|
||||||
|
const SystemSettingsSchema = new mongoose.Schema(
|
||||||
|
{
|
||||||
|
// How many months an event photo is retained before automatic cleanup.
|
||||||
|
photoRetentionMonths: { type: Number, default: 0, min: 0 },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
timestamps: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const SystemSettings = mongoose.model('SystemSettings', SystemSettingsSchema);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure the singleton settings document exists and return it.
|
||||||
|
*
|
||||||
|
* Lazily creates the document with defaults on first access so the retention
|
||||||
|
* cleanup always has a value to read even before any settings are saved.
|
||||||
|
*
|
||||||
|
* @returns {Promise<import('mongoose').Document>} the singleton settings doc
|
||||||
|
*/
|
||||||
|
async function getSettings() {
|
||||||
|
let settings = await SystemSettings.findById(SYSTEM_SETTINGS_ID);
|
||||||
|
if (!settings) {
|
||||||
|
settings = await SystemSettings.create({ _id: SYSTEM_SETTINGS_ID, photoRetentionMonths: 3 });
|
||||||
|
}
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the configured photo retention period in months.
|
||||||
|
*
|
||||||
|
* @returns {Promise<number>} retention in months (defaults to 3)
|
||||||
|
*/
|
||||||
|
async function getPhotoRetentionMonths() {
|
||||||
|
const settings = await getSettings();
|
||||||
|
const n = Number(settings.photoRetentionMonths);
|
||||||
|
if (!Number.isFinite(n) || n < 0) return 3;
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = SystemSettings;
|
||||||
|
module.exports.getSettings = getSettings;
|
||||||
|
module.exports.getPhotoRetentionMonths = getPhotoRetentionMonths;
|
||||||
|
module.exports.SYSTEM_SETTINGS_ID = SYSTEM_SETTINGS_ID;
|
||||||
|
|
@ -0,0 +1,129 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* Deletes event photos that are older than the configured retention period
|
||||||
|
* (`SystemSettings.photoRetentionMonths`, default 3). For each expired photo it:
|
||||||
|
* 1. Deletes the Rekognition FaceIds from the event's collection (DeleteFaces).
|
||||||
|
* 2. Deletes the S3 object via the existing `deleteEventPhotoFromS3()` helper.
|
||||||
|
* 3. Deletes the MongoDB EventPhoto document.
|
||||||
|
*
|
||||||
|
* The Rekognition collection itself is intentionally NOT deleted — the event may
|
||||||
|
* receive more photos later.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const EventPhoto = require('../models/EventPhoto');
|
||||||
|
const { getPhotoRetentionMonths } = require('../models/SystemSettings');
|
||||||
|
const rekognitionService = require('./rekognitionService');
|
||||||
|
const { deleteEventPhotoFromS3 } = require('../utils/s3EventPhotos');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute the cutoff `Date` before which photos are considered expired.
|
||||||
|
*
|
||||||
|
* @param {number} retentionMonths - retention in months
|
||||||
|
* @returns {Date} cutoff date
|
||||||
|
*/
|
||||||
|
function computeRetentionCutoff(retentionMonths) {
|
||||||
|
const months = Number(retentionMonths);
|
||||||
|
if (!Number.isFinite(months) || months < 0) return new Date(0);
|
||||||
|
if (months === 0) return new Date();
|
||||||
|
const now = new Date();
|
||||||
|
now.setUTCMonth(now.getUTCMonth() - months);
|
||||||
|
return now;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete a single expired photo's Rekognition faces, S3 object, and Mongo doc.
|
||||||
|
*
|
||||||
|
* Individual failures do not abort the whole run — each step is attempted and
|
||||||
|
* failures are counted so they can be logged/retried on the next run.
|
||||||
|
*
|
||||||
|
* @param {object} photo - lean EventPhoto document with _id, s3Key, collectionId, faces
|
||||||
|
* @returns {Promise<{ok: boolean, errors: string[]}>}
|
||||||
|
*/
|
||||||
|
async function cleanupPhoto(photo) {
|
||||||
|
const photoId = String(photo._id);
|
||||||
|
const errors = [];
|
||||||
|
const faceIds = (photo.faces || []).map((f) => f && f.faceId).filter(Boolean);
|
||||||
|
|
||||||
|
let canDeleteMongo = true;
|
||||||
|
|
||||||
|
// 1. Delete Rekognition FaceIds
|
||||||
|
if (photo.collectionId && faceIds.length > 0) {
|
||||||
|
try {
|
||||||
|
await rekognitionService.deleteFaces(photo.collectionId, faceIds);
|
||||||
|
} catch (err) {
|
||||||
|
errors.push(`rekognition: ${err?.message || String(err)}`);
|
||||||
|
canDeleteMongo = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Delete S3 object
|
||||||
|
if (photo.s3Key) {
|
||||||
|
try {
|
||||||
|
await deleteEventPhotoFromS3({ s3Key: photo.s3Key });
|
||||||
|
} catch (err) {
|
||||||
|
errors.push(`s3: ${err?.message || String(err)}`);
|
||||||
|
canDeleteMongo = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Delete MongoDB document only if previous steps succeeded
|
||||||
|
if (canDeleteMongo) {
|
||||||
|
try {
|
||||||
|
await EventPhoto.deleteOne({ _id: photo._id });
|
||||||
|
} catch (err) {
|
||||||
|
errors.push(`mongo: ${err?.message || String(err)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ok: errors.length === 0,
|
||||||
|
errors,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find and clean up all expired event photos.
|
||||||
|
*
|
||||||
|
* @returns {Promise<{deleted: number, failed: number, processed: number, retentionMonths: number}>}
|
||||||
|
*/
|
||||||
|
async function runPhotoCleanup() {
|
||||||
|
const retentionMonths = await getPhotoRetentionMonths();
|
||||||
|
const cutoff = computeRetentionCutoff(retentionMonths);
|
||||||
|
|
||||||
|
const expired = await EventPhoto.find({
|
||||||
|
createdAt: { $lt: cutoff },
|
||||||
|
}).lean();
|
||||||
|
|
||||||
|
let deleted = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
for (const photo of expired) {
|
||||||
|
try {
|
||||||
|
const result = await cleanupPhoto(photo);
|
||||||
|
if (result.ok) {
|
||||||
|
deleted += 1;
|
||||||
|
} else {
|
||||||
|
failed += 1;
|
||||||
|
console.error(
|
||||||
|
`[photoCleanup] Failed to clean up photo ${photo._id} (${photo.s3Key}): ${result.errors.join('; ')}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
failed += 1;
|
||||||
|
console.error(`[photoCleanup] Unexpected error cleaning photo ${photo._id}:`, err && err.message ? err.message : err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[photoCleanup] Retention run complete — retentionMonths=${retentionMonths} cutoff=${cutoff.toISOString()} ` +
|
||||||
|
`processed=${expired.length} deleted=${deleted} failed=${failed}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return { deleted, failed, processed: expired.length, retentionMonths };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
runPhotoCleanup,
|
||||||
|
computeRetentionCutoff,
|
||||||
|
cleanupPhoto,
|
||||||
|
};
|
||||||
|
|
@ -1,21 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* EventBuzz — Amazon Rekognition face-indexing service for event photos.
|
|
||||||
*
|
|
||||||
* Replaces the standalone AWS Lambda path (backend/src/lambda/eventPhotoIndex.js).
|
* Replaces the standalone AWS Lambda path (backend/src/lambda/eventPhotoIndex.js).
|
||||||
* The Hapi backend now calls Rekognition directly, immediately after an event
|
* The Hapi backend now calls Rekognition directly, immediately after an event
|
||||||
* photo is uploaded to S3 and persisted in MongoDB.
|
* photo is uploaded to S3 and persisted in MongoDB.
|
||||||
*
|
|
||||||
* Collection strategy:
|
|
||||||
* - CollectionId = eventId (the Mongo ObjectId of the Event).
|
|
||||||
* - Faces are scoped per event, so later face matching (e.g. SearchFacesByImage
|
|
||||||
* with CollectionId = eventId) is naturally scoped to a single event.
|
|
||||||
*
|
|
||||||
* Environment variables (kept consistent with backend/src/utils/s3EventPhotos.js):
|
|
||||||
* - AWS_EVENT_PICTURES_BUCKET S3 bucket (default: eventbuzz-pictures)
|
|
||||||
* - AWS_DEFAULT_REGION AWS region (default: ap-south-1)
|
|
||||||
* - AWS_ACCESS_KEY_ID optional, assumes an IAM role/permissions in prod
|
|
||||||
* - AWS_SECRET_ACCESS_KEY optional, assumes an IAM role/permissions in prod
|
|
||||||
* - AWS_SESSION_TOKEN optional, for temporary credentials
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const {
|
const {
|
||||||
|
|
@ -24,6 +10,7 @@ const {
|
||||||
CreateCollectionCommand,
|
CreateCollectionCommand,
|
||||||
DescribeCollectionCommand,
|
DescribeCollectionCommand,
|
||||||
SearchFacesByImageCommand,
|
SearchFacesByImageCommand,
|
||||||
|
DeleteFacesCommand,
|
||||||
} = require('@aws-sdk/client-rekognition');
|
} = require('@aws-sdk/client-rekognition');
|
||||||
const EventPhoto = require('../models/EventPhoto');
|
const EventPhoto = require('../models/EventPhoto');
|
||||||
|
|
||||||
|
|
@ -290,10 +277,47 @@ async function searchPhoto(eventId, options = {}) {
|
||||||
return { collectionExists: true, matches };
|
return { collectionExists: true, matches };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete specific face ids from a Rekognition collection.
|
||||||
|
*
|
||||||
|
* Used by the photo-retention cleanup to remove indexed faces of photos that are
|
||||||
|
* being deleted. Only the given face ids are removed — the collection itself is
|
||||||
|
* intentionally NOT deleted because the event may receive more photos later.
|
||||||
|
*
|
||||||
|
* If there are no face ids to delete, this is a no-op.
|
||||||
|
*
|
||||||
|
* @param {string} collectionId - the Rekognition CollectionId (the event id)
|
||||||
|
* @param {string[]} [faceIds] - array of face ids to delete
|
||||||
|
* @returns {Promise<{deleted: string[], failed: string[]}>}
|
||||||
|
*/
|
||||||
|
async function deleteFaces(collectionId, faceIds = []) {
|
||||||
|
const ids = Array.isArray(faceIds) ? faceIds.filter(Boolean) : [];
|
||||||
|
if (!collectionId || ids.length === 0) {
|
||||||
|
return { deleted: [], failed: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await getRekognitionClient().send(
|
||||||
|
new DeleteFacesCommand({
|
||||||
|
CollectionId: String(collectionId),
|
||||||
|
FaceIds: ids,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const deleted = (response.DeletedFaces || []).filter(Boolean);
|
||||||
|
const failed = ids.filter((id) => !deleted.includes(id));
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[rekognitionService] Deleted faces collection=${collectionId} requested=${ids.length} deleted=${deleted.length} failed=${failed.length}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return { deleted, failed };
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
getRekognitionClient,
|
getRekognitionClient,
|
||||||
ensureCollection,
|
ensureCollection,
|
||||||
indexPhoto,
|
indexPhoto,
|
||||||
searchPhoto,
|
searchPhoto,
|
||||||
|
deleteFaces,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue