From 01b6e22f83861479620bf39200d85f454b78d049 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 2 Aug 2021 18:41:30 -0600 Subject: [PATCH] Began scaffolding work for download manager --- backend/downloader.js | 284 ++++++++++++++++++++++++++++++++++++++++++ backend/test/tests.js | 35 ++++++ 2 files changed, 319 insertions(+) create mode 100644 backend/downloader.js diff --git a/backend/downloader.js b/backend/downloader.js new file mode 100644 index 0000000..4e44e86 --- /dev/null +++ b/backend/downloader.js @@ -0,0 +1,284 @@ + + +async function collectInfo(download_uid) { + const download = db_api.getRecord('download_queue', {uid: download_uid}); + + const url = download['url']; + const type = download['type']; + const options = download['options']; + const args = download['args']; + + // get video info prior to download + const info = await getVideoInfoByURL(url, args); + + if (!info) { + // info failed, record error and pause download + } + + // check if it fits into a category. If so, then get info again using new args + if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); + + // set custom output if the category has one and re-retrieve info so the download manager has the right file name + if (category && category['custom_output']) { + options.customOutput = category['custom_output']; + options.noRelativePath = true; + args = await generateArgs(url, type, options); + info = await getVideoInfoByURL(url, args); + + // must update args + await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args}); + } + + await db_api.updateRecord('download_queue', {uid: download_uid}, {remote_metadata: info}); +} + +async function downloadFileByURL_exec(url, type, options) { + const download = db_api.getRecord('download_queue', {uid: download_uid}); + + const url = download['url']; + const type = download['type']; + const options = download['options']; + const args = download['args']; + const category = download['category']; + let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix + if (options.user) { + let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + const user_path = path.join(usersFileFolder, options.user, type); + fs.ensureDirSync(user_path); + fileFolderPath = user_path + path.sep; + multiUserMode = { + user: options.user, + file_path: fileFolderPath + } + options.customFileFolderPath = fileFolderPath; + } + +} + +async function downloadFileByURL_exec_old(url, type, options, sessionID = null) { + return new Promise(async resolve => { + var date = Date.now(); + + // audio / video specific vars + var is_audio = type === 'audio'; + var ext = is_audio ? '.mp3' : '.mp4'; + var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; + let category = null; + + // prepend with user if needed + let multiUserMode = null; + if (options.user) { + let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + const user_path = path.join(usersFileFolder, options.user, type); + fs.ensureDirSync(user_path); + fileFolderPath = user_path + path.sep; + options.customFileFolderPath = fileFolderPath; + } + + options.downloading_method = 'exec'; + let downloadConfig = await generateArgs(url, type, options); + + // adds download to download helper + const download_uid = uuid(); + const session = sessionID ? sessionID : 'undeclared'; + let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session); + if (!session_downloads) { + session_downloads = {session_id: session}; + downloads.push(session_downloads); + } + session_downloads[download_uid] = { + uid: download_uid, + ui_uid: options.ui_uid, + downloading: true, + complete: false, + url: url, + type: type, + percent_complete: 0, + is_playlist: url.includes('playlist'), + timestamp_start: Date.now(), + filesize: null + }; + const download = session_downloads[download_uid]; + updateDownloads(); + + let download_checker = null; + + // get video info prior to download + let info = await getVideoInfoByURL(url, downloadConfig, download); + if (!info && url.includes('youtu')) { + resolve(false); + return; + } else if (info) { + // check if it fits into a category. If so, then get info again using new downloadConfig + if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); + + // set custom output if the category has one and re-retrieve info so the download manager has the right file name + if (category && category['custom_output']) { + options.customOutput = category['custom_output']; + options.noRelativePath = true; + downloadConfig = await generateArgs(url, type, options); + info = await getVideoInfoByURL(url, downloadConfig, download); + } + + // store info in download for future use + if (Array.isArray(info)) { + download['fileNames'] = []; + for (let info_obj of info) download['fileNames'].push(info_obj['_filename']); + } else { + download['_filename'] = info['_filename']; + } + download['filesize'] = utils.getExpectedFileSize(info); + download_checker = setInterval(() => checkDownloadPercent(download), 1000); + } + + // download file + youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { + if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) + + download['downloading'] = false; + download['timestamp_end'] = Date.now(); + var file_objs = []; + let new_date = Date.now(); + let difference = (new_date - date)/1000; + logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); + if (err) { + logger.error(err.stderr); + + download['error'] = err.stderr; + updateDownloads(); + resolve(false); + return; + } else if (output) { + if (output.length === 0 || output[0].length === 0) { + download['error'] = 'No output. Check if video already exists in your archive.'; + logger.warn(`No output received for video download, check if it exists in your archive.`) + updateDownloads(); + + resolve(false); + return; + } + var file_names = []; + for (let i = 0; i < output.length; i++) { + let output_json = null; + try { + output_json = JSON.parse(output[i]); + } catch(e) { + output_json = null; + } + + if (!output_json) { + continue; + } + + // get filepath with no extension + const filepath_no_extension = utils.removeFileExtension(output_json['_filename']); + + var full_file_path = filepath_no_extension + ext; + var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); + + if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 + && config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) { + let vodId = url.split('twitch.tv/videos/')[1]; + vodId = vodId.split('?')[0]; + twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user); + } + + // renames file if necessary due to bug + if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { + try { + fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); + logger.info('Renamed ' + file_name + '.webm to ' + file_name); + } catch(e) { + } + } + + if (type === 'audio') { + let tags = { + title: output_json['title'], + artist: output_json['artist'] ? output_json['artist'] : output_json['uploader'] + } + let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3'); + if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']); + } + + const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length); + const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null; + + if (options.cropFileSettings) { + await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext); + } + + // registers file in DB + const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings); + + // TODO: remove the following line + if (file_name) file_names.push(file_name); + + file_objs.push(file_obj); + } + + let is_playlist = file_names.length > 1; + + if (options.merged_string !== null && options.merged_string !== undefined) { + let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); + let diff = current_merged_archive.replace(options.merged_string, ''); + const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`); + fs.appendFileSync(archive_path, diff); + } + + download['complete'] = true; + download['fileNames'] = is_playlist ? file_names : [full_file_path] + updateDownloads(); + + let container = null; + + if (file_objs.length > 1) { + // create playlist + const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); + const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); + container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user); + } else if (file_objs.length === 1) { + container = file_objs[0]; + } else { + logger.error('Downloaded file failed to result in metadata object.'); + } + + resolve({ + file_uids: file_objs.map(file_obj => file_obj.uid), + container: container + }); + } + }); + }); +} + +// helper functions + +async function getVideoInfoByURL(url, args = [], download = null) { + return new Promise(resolve => { + // remove bad args + const new_args = [...args]; + + const archiveArgIndex = new_args.indexOf('--download-archive'); + if (archiveArgIndex !== -1) { + new_args.splice(archiveArgIndex, 2); + } + + // actually get info + youtubedl.getInfo(url, new_args, (err, output) => { + if (output) { + resolve(output); + } else { + logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`); + if (err.stderr) { + logger.error(`${err.stderr}`) + } + if (download) { + download['error'] = `Failed pre-check for video info: ${err}`; + updateDownloads(); + } + resolve(null); + } + }); + }); +} \ No newline at end of file diff --git a/backend/test/tests.js b/backend/test/tests.js index 17ae27c..9f19505 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -287,4 +287,39 @@ describe('Multi User', async function() { // }); // }); + describe('Downloader', function() { + const url = ''; + const options = { + ui_uid: uuid(), + user: 'admin' + } + + const download = { + url: url, + options: options, + type: 'video' + } + + beforeEach(async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords('download_queue'); + await db_api.insertRecordIntoTable('download_queue', download) + }); + + it('Get file info', async function() { + + }); + + it('Download file', async function() { + + }); + + it('Queue file', async function() { + + }); + + it('Pause file', async function() { + + }); + }); }); \ No newline at end of file