// This script is intended for migrating to the new Autumn release. // Please read all TODOs in this file as they will help guide you // to migrate your data properly. Please do Ctrl + F "TODO". import { MongoClient } from "mongodb"; /** * Map of tags to S3 bucket names * * TODO: if you've used AUTUMN_S3_BUCKET_PREFIX in the past * update the bucket names below to include the prefix * * NOTE: update `files.s3.default_bucket` in Revolt.toml! */ const BUCKET_MAP = { attachments: "attachments", avatars: "avatars", backgrounds: "backgrounds", icons: "icons", banners: "banners", emojis: "emojis", }; /** * Connection URL for MongoDB instance * * TODO: change if necessary */ const CONNECTION_URL = "mongodb://database"; const mongo = new MongoClient(CONNECTION_URL); await mongo.connect(); async function determineUploaderIdAndUse(f, v, i) { if (f.tag === "attachments" && v === "attachments") { if (typeof f.message_id !== "string") { console.warn(i, "No message id specified."); return null; } if (!objectLookup[f.message_id]) { objectLookup[f.message_id] = await mongo .db("revolt") .collection("messages") .findOne({ _id: f.message_id, }); } if (!objectLookup[f.message_id]) { console.warn(i, "Message", f.message_id, "doesn't exist anymore!"); return null; } return { uploaded_at: new Date(decodeTime(f.message_id)), uploader_id: objectLookup[f.message_id].author, used_for: { type: "Message", id: f.message_id, }, }; } else if (f.tag === "banners" && v === "banners") { if (typeof f.server_id !== "string") { console.warn(i, "No server id specified."); return null; } if (!objectLookup[f.server_id]) { objectLookup[f.server_id] = await mongo .db("revolt") .collection("servers") .findOne({ _id: f.server_id, }); } if (!objectLookup[f.server_id]) { console.warn(i, "Server", f.server_id, "doesn't exist anymore!"); return null; } return { uploaded_at: new Date(), uploader_id: objectLookup[f.server_id].owner, used_for: { type: "ServerBanner", id: f.server_id, }, }; } else if (f.tag === "emojis" && v === "emojis") { if (typeof f.object_id !== "string") { return null; } if (!objectLookup[f.object_id]) { objectLookup[f.object_id] = await mongo .db("revolt") .collection("emojis") .findOne({ _id: f.object_id, }); } if (!objectLookup[f.object_id]) { console.warn(i, "Emoji", f.object_id, "doesn't exist anymore!"); return null; } return { uploaded_at: new Date(decodeTime(f.object_id)), uploader_id: objectLookup[f.object_id].creator_id, used_for: { type: "Emoji", id: f.object_id, }, }; } else if (f.tag === "avatars" && v === "avatars") { if (typeof f.user_id !== "string") { return null; } if (!objectLookup[f.user_id]) { objectLookup[f.user_id] = await mongo .db("revolt") .collection("users") .findOne({ _id: f.user_id, }); } if (!objectLookup[f.user_id]) { console.warn(i, "User", f.user_id, "doesn't exist anymore!"); return null; } if (objectLookup[f.user_id].avatar?._id !== f._id) { console.warn( i, "Attachment no longer in use.", f._id, "for", f.user_id, "current:", objectLookup[f.user_id].avatar?._id ); return null; } return { uploaded_at: new Date(), uploader_id: f.user_id, used_for: { type: "UserAvatar", id: f.user_id, }, }; } else if (f.tag === "backgrounds" && v === "backgrounds") { if (typeof f.user_id !== "string") { return null; } if (!objectLookup[f.user_id]) { objectLookup[f.user_id] = await mongo .db("revolt") .collection("users") .findOne({ _id: f.user_id, }); } if (!objectLookup[f.user_id]) { console.warn(i, "User", f.user_id, "doesn't exist anymore!"); return null; } if (objectLookup[f.user_id].profile?.background?._id !== f._id) { console.warn( i, "Attachment no longer in use.", f._id, "for", f.user_id, "current:", objectLookup[f.user_id].profile?.background?._id ); return null; } return { uploaded_at: new Date(), uploader_id: f.user_id, used_for: { type: "UserProfileBackground", id: f.user_id, }, }; } else if (f.tag === "icons" && v === "icons") { if (typeof f.object_id !== "string") { return null; } // some bugged files at start // ... expensive to compute at worst case =( // so instead we can just disable it until everything is processed // then re-run on these! if (false) { objectLookup[f.object_id] = await mongo .db("revolt") .collection("users") .findOne({ _id: f.object_id, }); if (!objectLookup[f.object_id]) { console.warn(i, "No legacy match!"); return null; } return { uploaded_at: new Date(), uploader_id: f.object_id, used_for: { type: "LegacyGroupIcon", id: f.object_id, }, }; } if (!objectLookup[f.object_id]) { objectLookup[f.object_id] = await mongo .db("revolt") .collection("servers") .findOne({ _id: f.object_id, }); } if ( !objectLookup[f.object_id] || // heuristic for not server !objectLookup[f.object_id].channels ) { console.warn(i, "Server", f.object_id, "doesn't exist!"); if (!objectLookup[f.object_id]) { objectLookup[f.object_id] = await mongo .db("revolt") .collection("channels") .findOne({ _id: f.object_id, }); } if (!objectLookup[f.object_id]) { console.warn(i, "Channel", f.object_id, "doesn't exist!"); return null; } let server; const serverId = objectLookup[f.object_id].server; if (serverId) { server = objectLookup[serverId]; if (!server) { server = await mongo.db("revolt").collection("servers").findOne({ _id: serverId, }); console.info( i, "Couldn't find matching server for channel " + f.object_id + "!" ); if (!server) return null; objectLookup[serverId] = server; } } return { uploaded_at: new Date(), uploader_id: (server ?? objectLookup[f.object_id]).owner, used_for: { type: "ChannelIcon", id: f.object_id, }, }; } return { uploaded_at: new Date(), uploader_id: objectLookup[f.object_id].owner, used_for: { type: "ServerIcon", id: f.object_id, }, }; } else { throw ( "couldn't find uploader id for " + f._id + " expected " + v + " but got " + f.tag ); } } const dirs = [ "banners", "emojis", "avatars", "backgrounds", "icons", "attachments", // https://stackoverflow.com/a/18777877 ]; // === add `used_for` field to files const files_pt1 = await mongo .db("revolt") .collection("attachments") .find({ $or: [ { used_for: { $exists: false, }, }, { uploader_id: { $exists: false, }, }, { uploader_at: { $exists: false, }, }, ], }) .toArray(); let i = 1; for (const file of files_pt1) { console.info(i++, files_pt1.length, file); const meta = determineUploaderIdAndUse(file, file.tag, i); if (meta) { await mongo.db("revolt").collection("attachments").updateOne( { _id: file._id, }, { $set: meta, } ); } } // === set hash to id and create relevant objects const files_pt2 = await mongo .db("revolt") .collection("attachments") .find({ hash: { $exists: false, }, }) .toArray(); await mongo .db("revolt") .collection("attachment_hashes") .insertMany( files_pt2.map((file) => ({ _id: file._id, processed_hash: file._id, created_at: new Date(), bucket_id: BUCKET_MAP[file.tag], path: file._id, iv: "", // disable encryption for file metadata: file.metadata, content_type: file.content_type, size: file.size, })) ); for (const file of files_pt2) { await mongo .db("revolt") .collection("attachments") .updateOne( { _id: file._id, }, { $set: { hash: file._id, }, } ); }