diff --git a/backend/app.js b/backend/app.js index 43ab6bf..d4591c8 100644 --- a/backend/app.js +++ b/backend/app.js @@ -34,6 +34,7 @@ const categories_api = require('./categories'); const twitch_api = require('./twitch'); const youtubedl_api = require('./youtube-dl'); const archive_api = require('./archive'); +const files_api = require('./files'); var app = express(); @@ -173,10 +174,10 @@ async function checkMigrations() { if (!simplified_db_migration_complete) { logger.info('Beginning migration: 4.1->4.2+') let success = await simplifyDBFileStructure(); - success = success && await db_api.addMetadataPropertyToDB('view_count'); - success = success && await db_api.addMetadataPropertyToDB('description'); - success = success && await db_api.addMetadataPropertyToDB('height'); - success = success && await db_api.addMetadataPropertyToDB('abr'); + success = success && await files_api.addMetadataPropertyToDB('view_count'); + success = success && await files_api.addMetadataPropertyToDB('description'); + success = success && await files_api.addMetadataPropertyToDB('height'); + success = success && await files_api.addMetadataPropertyToDB('abr'); // sets migration to complete db.set('simplified_db_migration_complete', true).write(); if (success) { logger.info('4.1->4.2+ migration complete!'); } @@ -724,7 +725,7 @@ const optionalJwt = async function (req, res, next) { const uuid = using_body ? req.body.uuid : req.query.uuid; const uid = using_body ? req.body.uid : req.query.uid; const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id; - const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await db_api.getPlaylist(playlist_id, uuid, true); + const file = !playlist_id ? auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true); if (file) { req.can_watch = true; return next(); @@ -935,7 +936,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { const sub_id = req.body.sub_id; const uuid = req.isAuthenticated() ? req.user.uid : null; - const {files, file_count} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid); + const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid); res.send({ files: files, @@ -1101,7 +1102,7 @@ app.post('/api/incrementViewCount', async (req, res) => { uuid = req.user.uid; } - const file_obj = await db_api.getVideo(file_uid, uuid, sub_id); + const file_obj = await files_api.getVideo(file_uid, uuid, sub_id); const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0; const new_view_count = current_view_count + 1; @@ -1229,7 +1230,7 @@ app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => { let deleteForever = req.body.deleteForever; let file_uid = req.body.file_uid; - let success = await db_api.deleteFile(file_uid, deleteForever); + let success = await files_api.deleteFile(file_uid, deleteForever); if (success) { res.send({ @@ -1317,7 +1318,7 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { let playlistName = req.body.playlistName; let uids = req.body.uids; - const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null); + const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null); res.send({ new_playlist: new_playlist, @@ -1330,13 +1331,13 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null); let include_file_metadata = req.body.include_file_metadata; - const playlist = await db_api.getPlaylist(playlist_id, uuid); + const playlist = await files_api.getPlaylist(playlist_id, uuid); const file_objs = []; if (playlist && include_file_metadata) { for (let i = 0; i < playlist['uids'].length; i++) { const uid = playlist['uids'][i]; - const file_obj = await db_api.getVideo(uid, uuid); + const file_obj = await files_api.getVideo(uid, uuid); if (file_obj) file_objs.push(file_obj); // TODO: remove file from playlist if could not be found } @@ -1374,7 +1375,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => { playlist.uids.push(file_uid); - let success = await db_api.updatePlaylist(playlist); + let success = await files_api.updatePlaylist(playlist); res.send({ success: success }); @@ -1382,7 +1383,7 @@ app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => { app.post('/api/updatePlaylist', optionalJwt, async (req, res) => { let playlist = req.body.playlist; - let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid); + let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid); res.send({ success: success }); @@ -1412,7 +1413,7 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => { const blacklistMode = req.body.blacklistMode; let wasDeleted = false; - wasDeleted = await db_api.deleteFile(uid, blacklistMode); + wasDeleted = await files_api.deleteFile(uid, blacklistMode); res.send(wasDeleted); }); @@ -1444,7 +1445,7 @@ app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => { for (let i = 0; i < files.length; i++) { let wasDeleted = false; - wasDeleted = await db_api.deleteFile(files[i].uid, blacklistMode); + wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode); if (wasDeleted) { delete_count++; } @@ -1470,10 +1471,10 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => { if (playlist_id) { zip_file_generated = true; const playlist_files_to_download = []; - const playlist = await db_api.getPlaylist(playlist_id, uuid); + const playlist = await files_api.getPlaylist(playlist_id, uuid); for (let i = 0; i < playlist['uids'].length; i++) { const playlist_file_uid = playlist['uids'][i]; - const file_obj = await db_api.getVideo(playlist_file_uid, uuid); + const file_obj = await files_api.getVideo(playlist_file_uid, uuid); playlist_files_to_download.push(file_obj); } @@ -1487,7 +1488,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => { // generate zip file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download); } else { - const file_obj = await db_api.getVideo(uid, uuid, sub_id) + const file_obj = await files_api.getVideo(uid, uuid, sub_id) file_path_to_download = file_obj.path; } if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download); @@ -1634,7 +1635,7 @@ app.get('/api/stream', optionalJwt, async (req, res) => { const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); if (!multiUserMode || req.isAuthenticated() || req.can_watch) { - file_obj = await db_api.getVideo(uid, uuid, sub_id); + file_obj = await files_api.getVideo(uid, uuid, sub_id); if (file_obj) file_path = file_obj['path']; else file_path = null; } @@ -2082,7 +2083,7 @@ app.get('/api/rss', async function (req, res) { const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null; const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null; - const {files} = await db_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid); + const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid); const feed = new Feed({ title: 'Downloads', diff --git a/backend/db.js b/backend/db.js index 22e93ba..b577685 100644 --- a/backend/db.js +++ b/backend/db.js @@ -1,11 +1,11 @@ -var fs = require('fs-extra') -var path = require('path') +const fs = require('fs-extra') +const path = require('path') const { MongoClient } = require("mongodb"); const { uuid } = require('uuidv4'); const _ = require('lodash'); const config_api = require('./config'); -var utils = require('./utils') +const utils = require('./utils') const logger = require('./logger'); const low = require('lowdb') @@ -167,82 +167,9 @@ exports._connectToDB = async (custom_connection_string = null) => { } } -exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => { - if (!file_object) file_object = generateFileObject(file_path, type); - if (!file_object) { - logger.error(`Could not find associated JSON file for ${type} file ${file_path}`); - return false; - } - - utils.fixVideoMetadataPerms(file_path, type); - - // add thumbnail path - file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path); - - // if category exists, only include essential info - if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; - - // modify duration - if (cropFileSettings) { - file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; - } - - if (user_uid) file_object['user_uid'] = user_uid; - if (sub_id) file_object['sub_id'] = sub_id; - - const file_obj = await registerFileDBManual(file_object); - - // remove metadata JSON if needed - if (!config_api.getConfigItem('ytdl_include_metadata')) { - utils.deleteJSONFile(file_path, type) - } - - return file_obj; -} - -async function registerFileDBManual(file_object) { - // add additional info - file_object['uid'] = uuid(); - file_object['registered'] = Date.now(); - path_object = path.parse(file_object['path']); - file_object['path'] = path.format(path_object); - - await exports.insertRecordIntoTable('files', file_object, {path: file_object['path']}) - - return file_object; -} - -function generateFileObject(file_path, type) { - var jsonobj = utils.getJSON(file_path, type); - if (!jsonobj) { - return null; - } else if (!jsonobj['_filename']) { - logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`); - return null; - } - const ext = (type === 'audio') ? '.mp3' : '.mp4' - const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type); - // console. - var stats = fs.statSync(true_file_path); - - const file_id = utils.removeFileExtension(path.basename(file_path)); - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = utils.formatDateString(jsonobj.upload_date); - - var size = stats.size; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var isaudio = type === 'audio'; - var description = jsonobj.description; - var file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); - return file_obj; -} - -function getAppendedBasePathSub(sub, base_path) { - return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); +exports.setVideoProperty = async (file_uid, assignment_obj) => { + // TODO: check if video exists, throw error if not + await db_api.updateRecord('files', {uid: file_uid}, assignment_obj); } exports.getFileDirectoriesAndDBs = async () => { @@ -317,277 +244,6 @@ exports.getFileDirectoriesAndDBs = async () => { return dirs_to_check; } -exports.importUnregisteredFiles = async () => { - const imported_files = []; - const dirs_to_check = await exports.getFileDirectoriesAndDBs(); - - // run through check list and check each file to see if it's missing from the db - for (let i = 0; i < dirs_to_check.length; i++) { - const dir_to_check = dirs_to_check[i]; - // recursively get all files in dir's path - const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); - - for (let j = 0; j < files.length; j++) { - const file = files[j]; - - // check if file exists in db, if not add it - const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id}); - const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); - if (!file_is_registered) { - // add additional info - const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); - if (file_obj) { - imported_files.push(file_obj['uid']); - logger.verbose(`Added discovered file to the database: ${file.id}`); - } else { - logger.error(`Failed to import ${file['path']} automatically.`); - } - } - } - } - return imported_files; -} - -exports.addMetadataPropertyToDB = async (property_key) => { - try { - const dirs_to_check = await exports.getFileDirectoriesAndDBs(); - const update_obj = {}; - for (let i = 0; i < dirs_to_check.length; i++) { - const dir_to_check = dirs_to_check[i]; - - // recursively get all files in dir's path - const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); - for (let j = 0; j < files.length; j++) { - const file = files[j]; - if (file[property_key]) { - update_obj[file.uid] = {[property_key]: file[property_key]}; - } - } - } - - return await exports.bulkUpdateRecordsByKey('files', 'uid', update_obj); - } catch(err) { - logger.error(err); - return false; - } -} - -exports.createPlaylist = async (playlist_name, uids, user_uid = null) => { - const first_video = await exports.getVideo(uids[0]); - const thumbnailToUse = first_video['thumbnailURL']; - - let new_playlist = { - name: playlist_name, - uids: uids, - id: uuid(), - thumbnailURL: thumbnailToUse, - registered: Date.now(), - randomize_order: false - }; - - new_playlist.user_uid = user_uid ? user_uid : undefined; - - await exports.insertRecordIntoTable('playlists', new_playlist); - - const duration = await exports.calculatePlaylistDuration(new_playlist); - await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration}); - - return new_playlist; -} - -exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => { - let playlist = await exports.getRecord('playlists', {id: playlist_id}); - - if (!playlist) { - playlist = await exports.getRecord('categories', {uid: playlist_id}); - if (playlist) { - const uids = (await exports.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid); - playlist['uids'] = uids; - playlist['auto'] = true; - } - } - - // converts playlists to new UID-based schema - if (playlist && playlist['fileNames'] && !playlist['uids']) { - playlist['uids'] = []; - logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`); - for (let i = 0; i < playlist['fileNames'].length; i++) { - const fileName = playlist['fileNames'][i]; - const uid = await exports.getVideoUIDByID(fileName, user_uid); - if (uid) playlist['uids'].push(uid); - else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`); - } - exports.updatePlaylist(playlist, user_uid); - } - - // prevent unauthorized users from accessing the file info - if (require_sharing && !playlist['sharingEnabled']) return null; - - return playlist; -} - -exports.updatePlaylist = async (playlist) => { - let playlistID = playlist.id; - - const duration = await exports.calculatePlaylistDuration(playlist); - playlist.duration = duration; - - return await exports.updateRecord('playlists', {id: playlistID}, playlist); -} - -exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => { - let success = await exports.updateRecord('playlists', {id: playlist_id}, assignment_obj); - - if (!success) { - success = await exports.updateRecord('categories', {uid: playlist_id}, assignment_obj); - } - - if (!success) { - logger.error(`Could not find playlist or category with ID ${playlist_id}`); - } - - return success; -} - -exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => { - if (!playlist_file_objs) { - playlist_file_objs = []; - for (let i = 0; i < playlist['uids'].length; i++) { - const uid = playlist['uids'][i]; - const file_obj = await exports.getVideo(uid); - if (file_obj) playlist_file_objs.push(file_obj); - } - } - - return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); -} - -exports.deleteFile = async (uid, blacklistMode = false) => { - const file_obj = await exports.getVideo(uid); - const type = file_obj.isAudio ? 'audio' : 'video'; - const folderPath = path.dirname(file_obj.path); - const ext = type === 'audio' ? 'mp3' : 'mp4'; - const name = file_obj.id; - const filePathNoExtension = utils.removeFileExtension(file_obj.path); - - var jsonPath = `${file_obj.path}.info.json`; - var altJSONPath = `${filePathNoExtension}.info.json`; - var thumbnailPath = `${filePathNoExtension}.webp`; - var altThumbnailPath = `${filePathNoExtension}.jpg`; - - jsonPath = path.join(__dirname, jsonPath); - altJSONPath = path.join(__dirname, altJSONPath); - - let jsonExists = await fs.pathExists(jsonPath); - let thumbnailExists = await fs.pathExists(thumbnailPath); - - if (!jsonExists) { - if (await fs.pathExists(altJSONPath)) { - jsonExists = true; - jsonPath = altJSONPath; - } - } - - if (!thumbnailExists) { - if (await fs.pathExists(altThumbnailPath)) { - thumbnailExists = true; - thumbnailPath = altThumbnailPath; - } - } - - let fileExists = await fs.pathExists(file_obj.path); - - if (config_api.descriptors[uid]) { - try { - for (let i = 0; i < config_api.descriptors[uid].length; i++) { - config_api.descriptors[uid][i].destroy(); - } - } catch(e) { - - } - } - - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - // get id/extractor from JSON - - const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath)); - let retrievedID = null; - let retrievedExtractor = null; - if (info_json) { - retrievedID = info_json['id']; - retrievedExtractor = info_json['extractor']; - } - - // Remove file ID from the archive file, and write it to the blacklist (if enabled) - if (!blacklistMode) { - // workaround until a files_api is created (using archive_api would make a circular dependency) - await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id}); - // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id); - } - } - - if (jsonExists) await fs.unlink(jsonPath); - if (thumbnailExists) await fs.unlink(thumbnailPath); - - await exports.removeRecord('files', {uid: uid}); - - if (fileExists) { - await fs.unlink(file_obj.path); - if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) { - return false; - } else { - return true; - } - } else { - // TODO: tell user that the file didn't exist - return true; - } -} - -// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that -exports.getVideoUIDByID = async (file_id, uuid = null) => { - const file_obj = await exports.getRecord('files', {id: file_id}); - return file_obj ? file_obj['uid'] : null; -} - -exports.getVideo = async (file_uid) => { - return await exports.getRecord('files', {uid: file_uid}); -} - -exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => { - const filter_obj = {user_uid: uuid}; - const regex = true; - if (text_search) { - if (regex) { - filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'}; - } else { - filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) }; - } - } - - if (favorite_filter) { - filter_obj['favorite'] = true; - } - - if (sub_id) { - filter_obj['sub_id'] = sub_id; - } - - if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true; - else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false; - - const files = JSON.parse(JSON.stringify(await exports.getRecords('files', filter_obj, false, sort, range, text_search))); - const file_count = await exports.getRecords('files', filter_obj, true); - - return {files, file_count}; -} - -exports.setVideoProperty = async (file_uid, assignment_obj) => { - // TODO: check if video exists, throw error if not - await exports.updateRecord('files', {uid: file_uid}, assignment_obj); -} - // Basic DB functions // Create diff --git a/backend/downloader.js b/backend/downloader.js index b3915fa..8658502 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -13,6 +13,7 @@ const { create } = require('xmlbuilder2'); const categories_api = require('./categories'); const utils = require('./utils'); const db_api = require('./db'); +const files_api = require('./files'); const notifications_api = require('./notifications'); const archive_api = require('./archive'); @@ -385,7 +386,7 @@ async function downloadQueuedFile(download_uid) { } // registers file in DB - const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); + const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']); @@ -399,7 +400,7 @@ async function downloadQueuedFile(download_uid) { if (file_objs.length > 1) { // create playlist const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); - container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']); + container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']); } else if (file_objs.length === 1) { container = file_objs[0]; } else { diff --git a/backend/files.js b/backend/files.js new file mode 100644 index 0000000..14f6a92 --- /dev/null +++ b/backend/files.js @@ -0,0 +1,350 @@ +const fs = require('fs-extra') +const path = require('path') +const { uuid } = require('uuidv4'); + +const config_api = require('./config'); +const db_api = require('./db'); +const archive_api = require('./archive'); +const utils = require('./utils') +const logger = require('./logger'); + +exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => { + if (!file_object) file_object = generateFileObject(file_path, type); + if (!file_object) { + logger.error(`Could not find associated JSON file for ${type} file ${file_path}`); + return false; + } + + utils.fixVideoMetadataPerms(file_path, type); + + // add thumbnail path + file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path); + + // if category exists, only include essential info + if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; + + // modify duration + if (cropFileSettings) { + file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart; + } + + if (user_uid) file_object['user_uid'] = user_uid; + if (sub_id) file_object['sub_id'] = sub_id; + + const file_obj = await registerFileDBManual(file_object); + + // remove metadata JSON if needed + if (!config_api.getConfigItem('ytdl_include_metadata')) { + utils.deleteJSONFile(file_path, type) + } + + return file_obj; +} + +async function registerFileDBManual(file_object) { + // add additional info + file_object['uid'] = uuid(); + file_object['registered'] = Date.now(); + const path_object = path.parse(file_object['path']); + file_object['path'] = path.format(path_object); + + await db_api.insertRecordIntoTable('files', file_object, {path: file_object['path']}) + + return file_object; +} + +function generateFileObject(file_path, type) { + const jsonobj = utils.getJSON(file_path, type); + if (!jsonobj) { + return null; + } else if (!jsonobj['_filename']) { + logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`); + return null; + } + const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type); + // console. + const stats = fs.statSync(true_file_path); + + const file_id = utils.removeFileExtension(path.basename(file_path)); + const title = jsonobj.title; + const url = jsonobj.webpage_url; + const uploader = jsonobj.uploader; + const upload_date = utils.formatDateString(jsonobj.upload_date); + + const size = stats.size; + + const thumbnail = jsonobj.thumbnail; + const duration = jsonobj.duration; + const isaudio = type === 'audio'; + const description = jsonobj.description; + const file_obj = new utils.File(file_id, title, thumbnail, isaudio, duration, url, uploader, size, true_file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr); + return file_obj; +} + +exports.importUnregisteredFiles = async () => { + const imported_files = []; + const dirs_to_check = await db_api.getFileDirectoriesAndDBs(); + + // run through check list and check each file to see if it's missing from the db + for (let i = 0; i < dirs_to_check.length; i++) { + const dir_to_check = dirs_to_check[i]; + // recursively get all files in dir's path + const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); + + for (let j = 0; j < files.length; j++) { + const file = files[j]; + + // check if file exists in db, if not add it + const files_with_same_url = await db_api.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id}); + const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); + if (!file_is_registered) { + // add additional info + const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); + if (file_obj) { + imported_files.push(file_obj['uid']); + logger.verbose(`Added discovered file to the database: ${file.id}`); + } else { + logger.error(`Failed to import ${file['path']} automatically.`); + } + } + } + } + return imported_files; +} + +exports.addMetadataPropertyToDB = async (property_key) => { + try { + const dirs_to_check = await db_api.getFileDirectoriesAndDBs(); + const update_obj = {}; + for (let i = 0; i < dirs_to_check.length; i++) { + const dir_to_check = dirs_to_check[i]; + + // recursively get all files in dir's path + const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type, true); + for (let j = 0; j < files.length; j++) { + const file = files[j]; + if (file[property_key]) { + update_obj[file.uid] = {[property_key]: file[property_key]}; + } + } + } + + return await db_api.bulkUpdateRecordsByKey('files', 'uid', update_obj); + } catch(err) { + logger.error(err); + return false; + } +} + +exports.createPlaylist = async (playlist_name, uids, user_uid = null) => { + const first_video = await exports.getVideo(uids[0]); + const thumbnailToUse = first_video['thumbnailURL']; + + let new_playlist = { + name: playlist_name, + uids: uids, + id: uuid(), + thumbnailURL: thumbnailToUse, + registered: Date.now(), + randomize_order: false + }; + + new_playlist.user_uid = user_uid ? user_uid : undefined; + + await db_api.insertRecordIntoTable('playlists', new_playlist); + + const duration = await exports.calculatePlaylistDuration(new_playlist); + await db_api.updateRecord('playlists', {id: new_playlist.id}, {duration: duration}); + + return new_playlist; +} + +exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = false) => { + let playlist = await db_api.getRecord('playlists', {id: playlist_id}); + + if (!playlist) { + playlist = await db_api.getRecord('categories', {uid: playlist_id}); + if (playlist) { + const uids = (await db_api.getRecords('files', {'category.uid': playlist_id})).map(file => file.uid); + playlist['uids'] = uids; + playlist['auto'] = true; + } + } + + // converts playlists to new UID-based schema + if (playlist && playlist['fileNames'] && !playlist['uids']) { + playlist['uids'] = []; + logger.verbose(`Converting playlist ${playlist['name']} to new UID-based schema.`); + for (let i = 0; i < playlist['fileNames'].length; i++) { + const fileName = playlist['fileNames'][i]; + const uid = await exports.getVideoUIDByID(fileName, user_uid); + if (uid) playlist['uids'].push(uid); + else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`); + } + exports.updatePlaylist(playlist, user_uid); + } + + // prevent unauthorized users from accessing the file info + if (require_sharing && !playlist['sharingEnabled']) return null; + + return playlist; +} + +exports.updatePlaylist = async (playlist) => { + let playlistID = playlist.id; + + const duration = await exports.calculatePlaylistDuration(playlist); + playlist.duration = duration; + + return await db_api.updateRecord('playlists', {id: playlistID}, playlist); +} + +exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = null) => { + let success = await db_api.updateRecord('playlists', {id: playlist_id}, assignment_obj); + + if (!success) { + success = await db_api.updateRecord('categories', {uid: playlist_id}, assignment_obj); + } + + if (!success) { + logger.error(`Could not find playlist or category with ID ${playlist_id}`); + } + + return success; +} + +exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => { + if (!playlist_file_objs) { + playlist_file_objs = []; + for (let i = 0; i < playlist['uids'].length; i++) { + const uid = playlist['uids'][i]; + const file_obj = await exports.getVideo(uid); + if (file_obj) playlist_file_objs.push(file_obj); + } + } + + return playlist_file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); +} + +exports.deleteFile = async (uid, blacklistMode = false) => { + const file_obj = await exports.getVideo(uid); + const type = file_obj.isAudio ? 'audio' : 'video'; + const folderPath = path.dirname(file_obj.path); + const name = file_obj.id; + const filePathNoExtension = utils.removeFileExtension(file_obj.path); + + var jsonPath = `${file_obj.path}.info.json`; + var altJSONPath = `${filePathNoExtension}.info.json`; + var thumbnailPath = `${filePathNoExtension}.webp`; + var altThumbnailPath = `${filePathNoExtension}.jpg`; + + jsonPath = path.join(__dirname, jsonPath); + altJSONPath = path.join(__dirname, altJSONPath); + + let jsonExists = await fs.pathExists(jsonPath); + let thumbnailExists = await fs.pathExists(thumbnailPath); + + if (!jsonExists) { + if (await fs.pathExists(altJSONPath)) { + jsonExists = true; + jsonPath = altJSONPath; + } + } + + if (!thumbnailExists) { + if (await fs.pathExists(altThumbnailPath)) { + thumbnailExists = true; + thumbnailPath = altThumbnailPath; + } + } + + let fileExists = await fs.pathExists(file_obj.path); + + if (config_api.descriptors[uid]) { + try { + for (let i = 0; i < config_api.descriptors[uid].length; i++) { + config_api.descriptors[uid][i].destroy(); + } + } catch(e) { + + } + } + + let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive || file_obj.sub_id) { + // get id/extractor from JSON + + const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath)); + let retrievedID = null; + let retrievedExtractor = null; + if (info_json) { + retrievedID = info_json['id']; + retrievedExtractor = info_json['extractor']; + } + + // Remove file ID from the archive file, and write it to the blacklist (if enabled) + if (!blacklistMode) { + await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id) + } else { + const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id); + if (!exists_in_archive) { + await archive_api.addToArchive(retrievedExtractor, retrievedID, type, file_obj.title, file_obj.user_uid, file_obj.sub_id); + } + } + } + + if (jsonExists) await fs.unlink(jsonPath); + if (thumbnailExists) await fs.unlink(thumbnailPath); + + await db_api.removeRecord('files', {uid: uid}); + + if (fileExists) { + await fs.unlink(file_obj.path); + if (await fs.pathExists(jsonPath) || await fs.pathExists(file_obj.path)) { + return false; + } else { + return true; + } + } else { + // TODO: tell user that the file didn't exist + return true; + } +} + +// Video ID is basically just the file name without the base path and file extension - this method helps us get away from that +exports.getVideoUIDByID = async (file_id, uuid = null) => { + const file_obj = await db_api.getRecord('files', {id: file_id}); + return file_obj ? file_obj['uid'] : null; +} + +exports.getVideo = async (file_uid) => { + return await db_api.getRecord('files', {uid: file_uid}); +} + +exports.getAllFiles = async (sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid) => { + const filter_obj = {user_uid: uuid}; + const regex = true; + if (text_search) { + if (regex) { + filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'}; + } else { + filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) }; + } + } + + if (favorite_filter) { + filter_obj['favorite'] = true; + } + + if (sub_id) { + filter_obj['sub_id'] = sub_id; + } + + if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true; + else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false; + + const files = JSON.parse(JSON.stringify(await db_api.getRecords('files', filter_obj, false, sort, range, text_search))); + const file_count = await db_api.getRecords('files', filter_obj, true); + + return {files, file_count}; +} diff --git a/backend/tasks.js b/backend/tasks.js index 0cc465d..e7a3493 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -2,6 +2,7 @@ const db_api = require('./db'); const notifications_api = require('./notifications'); const youtubedl_api = require('./youtube-dl'); const archive_api = require('./archive'); +const files_api = require('./files'); const fs = require('fs-extra'); const logger = require('./logger'); @@ -20,7 +21,7 @@ const TASKS = { job: null }, missing_db_records: { - run: db_api.importUnregisteredFiles, + run: files_api.importUnregisteredFiles, title: 'Import missing DB records', job: null }, @@ -259,7 +260,7 @@ async function autoDeleteFiles(data) { logger.info(`Removing ${data['files_to_remove'].length} old files!`); for (let i = 0; i < data['files_to_remove'].length; i++) { const file_to_remove = data['files_to_remove'][i]; - await db_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files'])); + await files_api.deleteFile(file_to_remove['uid'], task_obj['options']['blacklist_files'] || (file_to_remove['sub_id'] && file_to_remove['blacklist_subscription_files'])); } } } diff --git a/backend/test/tests.js b/backend/test/tests.js index 0d2057c..3b79860 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -40,6 +40,7 @@ const utils = require('../utils'); const subscriptions_api = require('../subscriptions'); const archive_api = require('../archive'); const categories_api = require('../categories'); +const files_api = require('../files'); const fs = require('fs-extra'); const { uuid } = require('uuidv4'); const NodeID3 = require('node-id3'); @@ -356,7 +357,7 @@ describe('Multi User', async function() { }); const video_to_test = sample_video_json['uid']; it('Get video', async function() { - const video_obj = await db_api.getVideo(video_to_test); + const video_obj = await files_api.getVideo(video_to_test); assert(video_obj); }); @@ -374,12 +375,12 @@ describe('Multi User', async function() { }); describe('Zip generators', function() { it('Playlist zip generator', async function() { - const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test); + const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test); assert(playlist); const playlist_files_to_download = []; for (let i = 0; i < playlist['uids'].length; i++) { const uid = playlist['uids'][i]; - const playlist_file = await db_api.getVideo(uid, user_to_test); + const playlist_file = await files_api.getVideo(uid, user_to_test); playlist_files_to_download.push(playlist_file); } const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download); @@ -407,7 +408,7 @@ describe('Multi User', async function() { // const sub_to_test = ''; // const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e'; // it('Get video', async function() { - // const video_obj = db_api.getVideo(video_to_test, 'admin', ); + // const video_obj = files_api.getVideo(video_to_test, 'admin', ); // assert(video_obj); // }); diff --git a/src/app/app.component.html b/src/app/app.component.html index 5980509..a9853d7 100644 --- a/src/app/app.component.html +++ b/src/app/app.component.html @@ -21,7 +21,7 @@ person Profile -