From 0565cf24a6ee7a709c65021bdce72fbbeb63de02 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Mon, 27 Nov 2023 12:55:53 -0500 Subject: [PATCH] youtube-dl refactor (#956) * Consolidated all youtube-dl calls into one function * Downloads can now be cancelled and better "paused" * Removed node-youtube-dl dependency * Added ability to manually check a subscription, and to cancel a subscription check --------- Co-authored-by: Dedy Martadinata S --- Dockerfile | 26 +-- Public API v1.yaml | 63 +++++- backend/app.js | 73 +++--- backend/categories.js | 6 +- backend/consts.js | 3 +- backend/downloader.js | 122 +++++----- backend/package.json | 5 +- backend/subscriptions.js | 203 +++++++++-------- backend/test/tests.js | 162 ++++++++++++- backend/utils.js | 5 - backend/youtube-dl.js | 214 +++++++++--------- src/api-types/index.ts | 1 + .../models/CheckSubscriptionRequest.ts | 7 + src/api-types/models/Download.ts | 1 + src/api-types/models/Subscription.ts | 3 + src/api-types/models/UnsubscribeRequest.ts | 4 +- .../downloads/downloads.component.ts | 5 +- .../subscription-info-dialog.component.ts | 5 +- src/app/main/main.component.ts | 11 +- src/app/posts.services.ts | 17 +- .../subscription/subscription.component.html | 10 +- .../subscription/subscription.component.scss | 8 +- .../subscription/subscription.component.ts | 35 ++- 23 files changed, 655 insertions(+), 334 deletions(-) create mode 100644 src/api-types/models/CheckSubscriptionRequest.ts diff --git a/Dockerfile b/Dockerfile index 6c9b328..de05907 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,26 +19,22 @@ ENV USER=youtube ENV NO_UPDATE_NOTIFIER=true ENV PM2_HOME=/app/pm2 ENV ALLOW_CONFIG_MUTATIONS=true -# Directy fetch specific version -## https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_amd64.deb + +# Use NVM to get specific node version +ENV NODE_VERSION=16.14.2 RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \ apt update && \ - apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 && \ + apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 libatomic1 && \ apt clean && \ rm -rf /var/lib/apt/lists/* - RUN case ${TARGETPLATFORM} in \ - "linux/amd64") NODE_ARCH=amd64 ;; \ - "linux/arm") NODE_ARCH=armhf ;; \ - "linux/arm/v7") NODE_ARCH=armhf ;; \ - "linux/arm64") NODE_ARCH=arm64 ;; \ - esac \ - && curl -L https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_$NODE_ARCH.deb -o ./nodejs.deb && \ - apt update && \ - apt install -y ./nodejs.deb && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* &&\ - rm nodejs.deb; +RUN mkdir /usr/local/nvm +ENV PATH="/usr/local/nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +ENV NVM_DIR=/usr/local/nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} # Build frontend ARG BUILDPLATFORM diff --git a/Public API v1.yaml b/Public API v1.yaml index e49783d..9682671 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -293,6 +293,48 @@ paths: $ref: '#/components/schemas/UnsubscribeResponse' security: - Auth query parameter: [] + /api/checkSubscription: + post: + tags: + - subscriptions + summary: Run a check for videos for a subscription + description: Runs a subscription check + operationId: post-api-checksubscription + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CheckSubscriptionRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/cancelCheckSubscription: + post: + tags: + - subscriptions + summary: Cancels check for videos for a subscription + description: Cancels subscription check + operationId: post-api-checksubscription + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CheckSubscriptionRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] /api/deleteSubscriptionFile: post: tags: @@ -1981,11 +2023,11 @@ components: type: string UnsubscribeRequest: required: - - sub + - sub_id type: object properties: - sub: - $ref: '#/components/schemas/SubscriptionRequestData' + sub_id: + type: string deleteMode: type: boolean description: Defaults to false @@ -1998,6 +2040,13 @@ components: type: boolean error: type: string + CheckSubscriptionRequest: + required: + - sub_id + type: object + properties: + sub_id: + type: string DeleteAllFilesResponse: type: object properties: @@ -2683,6 +2732,8 @@ components: type: boolean paused: type: boolean + cancelled: + type: boolean finished_step: type: boolean url: @@ -2841,6 +2892,8 @@ components: nullable: true isPlaylist: type: boolean + child_process: + type: object archive: type: string timerange: @@ -2849,6 +2902,10 @@ components: type: string custom_output: type: string + downloading: + type: boolean + paused: + type: boolean videos: type: array items: diff --git a/backend/app.js b/backend/app.js index 1077665..f1cb8e6 100644 --- a/backend/app.js +++ b/backend/app.js @@ -20,11 +20,6 @@ const ps = require('ps-node'); const Feed = require('feed').Feed; const session = require('express-session'); -// needed if bin/details somehow gets deleted -if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) - -const youtubedl = require('youtube-dl'); - const logger = require('./logger'); const config_api = require('./config.js'); const downloader_api = require('./downloader'); @@ -536,7 +531,7 @@ async function loadConfig() { // set downloading to false let subscriptions = await subscriptions_api.getAllSubscriptions(); subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub)); - subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); + subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false, child_process: null}); // runs initially, then runs every ${subscriptionCheckInterval} seconds const watchSubscriptionsInterval = function() { watchSubscriptions(); @@ -657,36 +652,20 @@ function generateEnvVarConfigItem(key) { // currently only works for single urls async function getUrlInfos(url) { - let startDate = Date.now(); - let result = []; - return new Promise(resolve => { - youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => { - let new_date = Date.now(); - let difference = (new_date - startDate)/1000; - logger.debug(`URL info retrieval delay: ${difference} seconds.`); - if (err) { - logger.error(`Error during retrieving formats for ${url}: ${err}`); - resolve(null); - } - let try_putput = null; - try { - try_putput = JSON.parse(output); - result = try_putput; - } catch(e) { - logger.error(`Failed to retrieve available formats for url: ${url}`); - } - resolve(result); - }); - }); + const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, ['--dump-json']); + if (!parsed_output || parsed_output.length !== 1) { + logger.error(`Failed to retrieve available formats for url: ${url}`); + if (err) logger.error(err); + return null; + } + return parsed_output[0]; } // youtube-dl functions async function startYoutubeDL() { // auto update youtube-dl - youtubedl_api.verifyBinaryExistsLinux(); - const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); - if (update_available) await youtubedl_api.updateYoutubeDL(update_available); + await youtubedl_api.checkForYoutubeDLUpdate(); } app.use(function(req, res, next) { @@ -1212,10 +1191,10 @@ app.post('/api/subscribe', optionalJwt, async (req, res) => { app.post('/api/unsubscribe', optionalJwt, async (req, res) => { let deleteMode = req.body.deleteMode - let sub = req.body.sub; + let sub_id = req.body.sub_id; let user_uid = req.isAuthenticated() ? req.user.uid : null; - let result_obj = subscriptions_api.unsubscribe(sub, deleteMode, user_uid); + let result_obj = subscriptions_api.unsubscribe(sub_id, deleteMode, user_uid); if (result_obj.success) { res.send({ success: result_obj.success @@ -1305,6 +1284,36 @@ app.post('/api/updateSubscription', optionalJwt, async (req, res) => { }); }); +app.post('/api/checkSubscription', optionalJwt, async (req, res) => { + let sub_id = req.body.sub_id; + let user_uid = req.isAuthenticated() ? req.user.uid : null; + + const success = subscriptions_api.getVideosForSub(sub_id, user_uid); + res.send({ + success: success + }); +}); + +app.post('/api/cancelCheckSubscription', optionalJwt, async (req, res) => { + let sub_id = req.body.sub_id; + let user_uid = req.isAuthenticated() ? req.user.uid : null; + + const success = subscriptions_api.cancelCheckSubscription(sub_id, user_uid); + res.send({ + success: success + }); +}); + +app.post('/api/cancelSubscriptionCheck', optionalJwt, async (req, res) => { + let sub_id = req.body.sub_id; + let user_uid = req.isAuthenticated() ? req.user.uid : null; + + const success = subscriptions_api.getVideosForSub(sub_id, user_uid); + res.send({ + success: success + }); +}); + app.post('/api/getSubscriptions', optionalJwt, async (req, res) => { let user_uid = req.isAuthenticated() ? req.user.uid : null; diff --git a/backend/categories.js b/backend/categories.js index 2faebfb..c76368b 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -32,10 +32,8 @@ async function categorize(file_jsons) { return null; } - for (let i = 0; i < file_jsons.length; i++) { - const file_json = file_jsons[i]; - for (let j = 0; j < categories.length; j++) { - const category = categories[j]; + for (const file_json of file_jsons) { + for (const category of categories) { const rules = category['rules']; // if rules for current category apply, then that is the selected category diff --git a/backend/consts.js b/backend/consts.js index e121dad..b092ca5 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -269,7 +269,8 @@ exports.AVAILABLE_PERMISSIONS = [ 'tasks_manager' ]; -exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details' +exports.DETAILS_BIN_PATH = 'appdata/youtube-dl.json' +exports.OUTDATED_YOUTUBEDL_VERSION = "2020.00.00"; // args that have a value after it (e.g. -o or -f ) const YTDL_ARGS_WITH_VALUES = [ diff --git a/backend/downloader.js b/backend/downloader.js index d37a275..c23b23e 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -4,8 +4,6 @@ const path = require('path'); const NodeID3 = require('node-id3') const Mutex = require('async-mutex').Mutex; -const youtubedl = require('youtube-dl'); - const logger = require('./logger'); const youtubedl_api = require('./youtube-dl'); const config_api = require('./config'); @@ -21,6 +19,8 @@ const archive_api = require('./archive'); const mutex = new Mutex(); let should_check_downloads = true; +const download_to_child_process = {}; + if (db_api.database_initialized) { exports.setupDownloads(); } else { @@ -84,8 +84,11 @@ exports.pauseDownload = async (download_uid) => { } else if (download['finished']) { logger.info(`Download ${download_uid} could not be paused before completing.`); return false; + } else { + logger.info(`Pausing download ${download_uid}`); } + killActiveDownload(download); return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false}); } @@ -120,16 +123,23 @@ exports.cancelDownload = async (download_uid) => { } else if (download['finished']) { logger.info(`Download ${download_uid} could not be cancelled before completing.`); return false; + } else { + logger.info(`Cancelling download ${download_uid}`); } - return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false}); + + killActiveDownload(download); + await handleDownloadError(download_uid, 'Cancelled', 'cancelled'); + return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true}); } exports.clearDownload = async (download_uid) => { return await db_api.removeRecord('download_queue', {uid: download_uid}); } -async function handleDownloadError(download, error_message, error_type = null) { - if (!download || !download['uid']) return; +async function handleDownloadError(download_uid, error_message, error_type = null) { + if (!download_uid) return; + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (!download || download['error']) return; notifications_api.sendDownloadErrorNotification(download, download['user_uid'], error_message, error_type); await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type}); } @@ -180,14 +190,14 @@ async function checkDownloads() { if (waiting_download['sub_id']) { const sub_missing = !(await db_api.getRecord('subscriptions', {id: waiting_download['sub_id']})); if (sub_missing) { - handleDownloadError(waiting_download, `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing'); + handleDownloadError(waiting_download['uid'], `Download failed as subscription with id '${waiting_download['sub_id']}' is missing!`, 'sub_id_missing'); continue; } } // move to next step running_downloads_count++; if (waiting_download['step_index'] === 0) { - collectInfo(waiting_download['uid']); + exports.collectInfo(waiting_download['uid']); } else if (waiting_download['step_index'] === 1) { exports.downloadQueuedFile(waiting_download['uid']); } @@ -195,7 +205,15 @@ async function checkDownloads() { } } -async function collectInfo(download_uid) { +function killActiveDownload(download) { + const child_process = download_to_child_process[download['uid']]; + if (download['step_index'] === 2 && child_process) { + youtubedl_api.killYoutubeDLProcess(child_process); + delete download_to_child_process[download['uid']]; + } +} + +exports.collectInfo = async (download_uid) => { const download = await db_api.getRecord('download_queue', {uid: download_uid}); if (download['paused']) { return; @@ -218,21 +236,21 @@ async function collectInfo(download_uid) { // get video info prior to download let info = download['prefetched_info'] ? download['prefetched_info'] : await exports.getVideoInfoByURL(url, args, download_uid); - if (!info) { + if (!info || info.length === 0) { // info failed, error presumably already recorded return; } // in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive && !options.ignoreArchive) { - const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']); + if (useYoutubeDLArchive && !options.ignoreArchive && info.length === 1) { + const info_obj = info[0]; + const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info_obj['id'], type, download['user_uid'], download['sub_id']); if (exists_in_archive) { - const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`; + const error = `File '${info_obj['title']}' already exists in archive! Disable the archive or override to continue downloading.`; logger.warn(error); if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error, 'exists_in_archive'); + await handleDownloadError(download_uid, error, 'exists_in_archive'); return; } } @@ -241,7 +259,7 @@ async function collectInfo(download_uid) { let category = null; // check if it fits into a category. If so, then get info again using new args - if (info.length === 0 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); + if (info.length === 1 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info); // set custom output if the category has one and re-retrieve info so the download manager has the right file name if (category && category['custom_output']) { @@ -262,20 +280,20 @@ async function collectInfo(download_uid) { // store info in download for future use for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename'])); - const playlist_title = info.length > 0 ? info[0]['playlist_title'] || info[0]['playlist'] : null; + const title = info.length > 1 ? info[0]['playlist_title'] || info[0]['playlist'] : info[0]['title']; await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args, finished_step: true, running: false, options: options, files_to_check_for_progress: files_to_check_for_progress, expected_file_size: expected_file_size, - title: playlist_title ? playlist_title : info['title'], + title: title, category: stripped_category, prefetched_info: null }); } -exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => { +exports.downloadQueuedFile = async(download_uid, customDownloadHandler = null) => { const download = await db_api.getRecord('download_queue', {uid: download_uid}); if (download['paused']) { return; @@ -305,21 +323,25 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000); const file_objs = []; // download file - const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod); + let {child_process, callback} = await youtubedl_api.runYoutubeDL(url, args, customDownloadHandler); + if (child_process) download_to_child_process[download['uid']] = child_process; + const {parsed_output, err} = await callback; clearInterval(download_checker); let end_time = Date.now(); let difference = (end_time - start_time)/1000; logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); if (!parsed_output) { - logger.error(err.stderr); - await handleDownloadError(download, err.stderr, 'unknown_error'); + const errored_download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (errored_download && errored_download['paused']) return; + logger.error(err.toString()); + await handleDownloadError(download_uid, err.toString(), 'unknown_error'); resolve(false); return; } else if (parsed_output) { if (parsed_output.length === 0 || parsed_output[0].length === 0) { // ERROR! const error_message = `No output received for video download, check if it exists in your archive.`; - await handleDownloadError(download, error_message, 'no_output'); + await handleDownloadError(download_uid, error_message, 'no_output'); logger.warn(error_message); resolve(false); return; @@ -385,14 +407,13 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec if (file_objs.length > 1) { // create playlist - const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); - container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']); + container = await files_api.createPlaylist(download['title'], file_objs.map(file_obj => file_obj.uid), download['user_uid']); } else if (file_objs.length === 1) { container = file_objs[0]; } else { const error_message = 'Downloaded file failed to result in metadata object.'; logger.error(error_message); - await handleDownloadError(download, error_message, 'no_metadata'); + await handleDownloadError(download_uid, error_message, 'no_metadata'); } const file_uids = file_objs.map(file_obj => file_obj.uid); @@ -405,7 +426,7 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec // helper functions exports.generateArgs = async (url, type, options, user_uid = null, simulated = false) => { - const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); if (!simulated && (default_downloader === 'youtube-dl' || default_downloader === 'youtube-dlc')) { logger.warn('It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.') @@ -536,34 +557,30 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f } exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { - return new Promise(resolve => { - // remove bad args - const temp_args = utils.filterArgs(args, ['--no-simulate']); - const new_args = [...temp_args]; - - const archiveArgIndex = new_args.indexOf('--download-archive'); - if (archiveArgIndex !== -1) { - new_args.splice(archiveArgIndex, 2); - } + // remove bad args + const temp_args = utils.filterArgs(args, ['--no-simulate']); + const new_args = [...temp_args]; - new_args.push('--dump-json'); + const archiveArgIndex = new_args.indexOf('--download-archive'); + if (archiveArgIndex !== -1) { + new_args.splice(archiveArgIndex, 2); + } - youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => { - const parsed_output = utils.parseOutputJSON(output, err); - if (parsed_output) { - resolve(parsed_output); - } else { - let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; - if (err.stderr) error_message += `\n\n${err.stderr}`; - logger.error(error_message); - if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error_message, 'info_retrieve_failed'); - } - resolve(null); - } - }); - }); + new_args.push('--dump-json'); + + let {callback} = await youtubedl_api.runYoutubeDL(url, new_args); + const {parsed_output, err} = await callback; + if (!parsed_output || parsed_output.length === 0) { + let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; + if (err.stderr) error_message += `\n\n${err.stderr}`; + logger.error(error_message); + if (download_uid) { + await handleDownloadError(download_uid, error_message, 'info_retrieve_failed'); + } + return null; + } + + return parsed_output; } function filterArgs(args, isAudio) { @@ -582,6 +599,7 @@ async function checkDownloadPercent(download_uid) { */ const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (!download) return; const files_to_check_for_progress = download['files_to_check_for_progress']; const resulting_file_size = download['expected_file_size']; diff --git a/backend/package.json b/backend/package.json index 11f0e92..1366e7f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -33,6 +33,7 @@ "command-exists": "^1.2.9", "compression": "^1.7.4", "config": "^3.2.3", + "execa": "^5.1.1", "express": "^4.18.2", "express-session": "^1.17.3", "feed": "^4.2.2", @@ -61,10 +62,10 @@ "read-last-lines": "^1.7.2", "rxjs": "^7.3.0", "shortid": "^2.2.15", + "tree-kill": "^1.2.2", "unzipper": "^0.10.10", "uuidv4": "^6.2.13", "winston": "^3.7.2", - "xmlbuilder2": "^3.0.2", - "youtube-dl": "^3.0.2" + "xmlbuilder2": "^3.0.2" } } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 547ac80..034d652 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -1,7 +1,7 @@ const fs = require('fs-extra'); const path = require('path'); -const youtubedl = require('youtube-dl'); +const youtubedl_api = require('./youtube-dl'); const config_api = require('./config'); const archive_api = require('./archive'); const utils = require('./utils'); @@ -39,7 +39,7 @@ exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => { exports.writeSubscriptionMetadata(sub); if (success) { - if (!sub.paused) exports.getVideosForSub(sub, user_uid); + if (!sub.paused) exports.getVideosForSub(sub.id); } else { logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') } @@ -63,55 +63,41 @@ async function getSubscriptionInfo(sub) { } } - return new Promise(async resolve => { - youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => { - if (debugMode) { - logger.info('Subscribe: got info for subscription ' + sub.id); + let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await callback; + if (err) { + logger.error(err.stderr); + return false; + } + logger.verbose('Subscribe: got info for subscription ' + sub.id); + for (const output_json of parsed_output) { + if (!output_json) { + continue; + } + + if (!sub.name) { + if (sub.isPlaylist) { + sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist; + } else { + sub.name = output_json.uploader; } - if (err) { - logger.error(err.stderr); - resolve(false); - } else if (output) { - if (output.length === 0 || (output.length === 1 && output[0] === '')) { - logger.verbose('Could not get info for ' + sub.id); - resolve(false); - } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - if (!output_json) { - continue; - } - if (!sub.name) { - if (sub.isPlaylist) { - sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist; - } else { - sub.name = output_json.uploader; - } - // if it's now valid, update - if (sub.name) { - let sub_name = sub.name; - const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid}); - if (sub_name_exists) sub_name += ` - ${sub.id}`; - await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name}); - } - } - - // TODO: get even more info - - resolve(true); - } - resolve(false); + // if it's now valid, update + if (sub.name) { + let sub_name = sub.name; + const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid}); + if (sub_name_exists) sub_name += ` - ${sub.id}`; + await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name}); } - }); - }); + } + + return true; + } + + return false; } -exports.unsubscribe = async (sub, deleteMode, user_uid = null) => { +exports.unsubscribe = async (sub_id, deleteMode, user_uid = null) => { + const sub = await exports.getSubscription(sub_id); let basePath = null; if (user_uid) basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); @@ -134,6 +120,7 @@ exports.unsubscribe = async (sub, deleteMode, user_uid = null) => { } } + await killSubDownloads(sub_id, true); await db_api.removeRecord('subscriptions', {id: id}); await db_api.removeAllRecords('files', {sub_id: id}); @@ -218,12 +205,18 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul } } -exports.getVideosForSub = async (sub, user_uid = null) => { - const latest_sub_obj = await exports.getSubscription(sub.id); - if (!latest_sub_obj || latest_sub_obj['downloading']) { +exports.getVideosForSub = async (sub_id) => { + const sub = await exports.getSubscription(sub_id); + if (!sub || sub['downloading']) { return false; } + _getVideosForSub(sub); + return true; +} + +async function _getVideosForSub(sub) { + const user_uid = sub['user_uid']; updateSubscriptionProperty(sub, {downloading: true}, user_uid); // get basePath @@ -241,33 +234,26 @@ exports.getVideosForSub = async (sub, user_uid = null) => { // get videos logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`); - return new Promise(async resolve => { - youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { - // cleanup - updateSubscriptionProperty(sub, {downloading: false}, user_uid); - - // remove temporary archive file if it exists - const archive_path = path.join(appendedBasePath, 'archive.txt'); - const archive_exists = await fs.pathExists(archive_path); - if (archive_exists) { - await fs.unlink(archive_path); - } + let {child_process, callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + updateSubscriptionProperty(sub, {child_process: child_process}, user_uid); + const {parsed_output, err} = await callback; + updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid); + if (!parsed_output) { + logger.error('Subscription check failed!'); + if (err) logger.error(err); + return null; + } - logger.verbose('Subscription: finished check for ' + sub.name); - const parsed_output = utils.parseOutputJSON(output, err); - if (!parsed_output) { - logger.error('Subscription check failed!'); - resolve(null); - return; - } - const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid); - resolve(files_to_download); - return; - }); - }, err => { - logger.error(err); - updateSubscriptionProperty(sub, {downloading: false}, user_uid); - }); + // remove temporary archive file if it exists + const archive_path = path.join(appendedBasePath, 'archive.txt'); + const archive_exists = await fs.pathExists(archive_path); + if (archive_exists) { + await fs.unlink(archive_path); + } + + logger.verbose('Subscription: finished check for ' + sub.name); + const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid); + return files_to_download; } async function handleOutputJSON(output_jsons, sub, user_uid) { @@ -388,7 +374,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push('-r', rate_limit); } - const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); if (default_downloader === 'yt-dlp') { downloadConfig.push('--no-clean-info-json'); } @@ -419,8 +405,37 @@ async function getFilesToDownload(sub, output_jsons) { return files_to_download; } +exports.cancelCheckSubscription = async (sub_id) => { + const sub = await exports.getSubscription(sub_id); + if (!sub['downloading'] && !sub['child_process']) { + logger.error('Failed to cancel subscription check, verify that it is still running!'); + return false; + } + + // if check is ongoing + if (sub['child_process']) { + const child_process = sub['child_process']; + youtubedl_api.killYoutubeDLProcess(child_process); + } + + // cancel activate video downloads + await killSubDownloads(sub_id); + + return true; +} + +async function killSubDownloads(sub_id, remove_downloads = false) { + const sub_downloads = await db_api.getRecords('download_queue', {sub_id: sub_id}); + for (const sub_download of sub_downloads) { + if (sub_download['running']) + await downloader_api.cancelDownload(sub_download['uid']); + if (remove_downloads) + await db_api.removeRecord('download_queue', {uid: sub_download['uid']}); + } +} exports.getSubscriptions = async (user_uid = null) => { + // TODO: fix issue where the downloading property may not match getSubscription() return await db_api.getRecords('subscriptions', {user_uid: user_uid}); } @@ -499,24 +514,22 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path); logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`); // simulate a download to verify that a better version exists - youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => { - if (err) { - // video is not available anymore for whatever reason - } else if (output) { - const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; - if (output[metric_to_compare] > file_obj[metric_to_compare]) { - // download new video as the simulated one is better - youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => { - if (err) { - logger.verbose(`Failed to download better version of video ${file_obj['id']}`); - } else if (output) { - logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`); - await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}); - } - }); - } - } - }); + + const info = await downloader_api.getVideoInfoByURL(file_obj['url'], downloadConfig); + if (info && info.length === 1) { + const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height'; + if (info[metric_to_compare] > file_obj[metric_to_compare]) { + // download new video as the simulated one is better + let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig); + const {parsed_output, err} = await callback; + if (err) { + logger.verbose(`Failed to download better version of video ${file_obj['id']}`); + } else if (parsed_output) { + logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${info[metric_to_compare]}`); + await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: info[metric_to_compare]}); + } + } + } await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}); } diff --git a/backend/test/tests.js b/backend/test/tests.js index 54617a0..2f286ec 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -4,6 +4,9 @@ const low = require('lowdb') const winston = require('winston'); const path = require('path'); const util = require('util'); +const fs = require('fs-extra'); +const { uuid } = require('uuidv4'); +const NodeID3 = require('node-id3'); const exec = util.promisify(require('child_process').exec); const FileSync = require('lowdb/adapters/FileSync'); @@ -44,9 +47,7 @@ const categories_api = require('../categories'); const files_api = require('../files'); const youtubedl_api = require('../youtube-dl'); const config_api = require('../config'); -const fs = require('fs-extra'); -const { uuid } = require('uuidv4'); -const NodeID3 = require('node-id3'); +const CONSTS = require('../consts'); db_api.initialize(db, users_db, 'local_db_test.json'); @@ -441,11 +442,34 @@ describe('Multi User', async function() { describe('Downloader', function() { const downloader_api = require('../downloader'); const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI'; + const playlist_url = 'https://www.youtube.com/playlist?list=PLbZT16X07RLhqK-ZgSkRuUyiz9B_WLdNK'; const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; const options = { ui_uid: uuid() } + async function createCategory(url) { + // get info + const args = await downloader_api.generateArgs(url, 'video', options, null, true); + const [info] = await downloader_api.getVideoInfoByURL(url, args); + + // create category + await db_api.removeAllRecords('categories'); + const new_category = { + name: 'test_category', + uid: uuid(), + rules: [], + custom_output: '' + }; + await db_api.insertRecordIntoTable('categories', new_category); + await db_api.pushToRecordsArray('categories', {name: 'test_category'}, 'rules', { + preceding_operator: null, + comparator: 'includes', + property: 'title', + value: info['title'] + }); + } + before(async function() { const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); if (update_available) await youtubedl_api.updateYoutubeDL(update_available); @@ -455,6 +479,7 @@ describe('Downloader', function() { beforeEach(async function() { // await db_api.connectToDB(); await db_api.removeAllRecords('download_queue'); + config_api.setConfigItem('ytdl_allow_playlist_categorization', true); }); it('Get file info', async function() { @@ -480,6 +505,32 @@ describe('Downloader', function() { assert(success); }); + it('Downloader - categorize', async function() { + this.timeout(300000); + await createCategory(url); + // collect info + const returned_download = await downloader_api.createDownload(url, 'video', options); + await downloader_api.collectInfo(returned_download['uid']); + assert(returned_download['category']); + assert(returned_download['category']['name'] === 'test_category'); + }); + + it('Downloader - categorize playlist', async function() { + this.timeout(300000); + await createCategory(playlist_url); + // collect info + const returned_download_pass = await downloader_api.createDownload(playlist_url, 'video', options); + await downloader_api.collectInfo(returned_download_pass['uid']); + assert(returned_download_pass['category']); + assert(returned_download_pass['category']['name'] === 'test_category'); + + // test with playlist categorization disabled + config_api.setConfigItem('ytdl_allow_playlist_categorization', false); + const returned_download_fail = await downloader_api.createDownload(playlist_url, 'video', options); + await downloader_api.collectInfo(returned_download_fail['uid']); + assert(!returned_download_fail['category']); + }); + it('Tag file', async function() { const success = await generateEmptyAudioFile('test/sample_mp3.mp3'); const audio_path = './test/sample_mp3.mp3'; @@ -552,7 +603,7 @@ describe('Downloader', function() { }); describe('Twitch', async function () { const twitch_api = require('../twitch'); - const example_vod = '1710641401'; + const example_vod = '1790315420'; it('Download VOD chat', async function() { this.timeout(300000); if (!fs.existsSync('TwitchDownloaderCLI')) { @@ -574,6 +625,105 @@ describe('Downloader', function() { }); }); +describe('youtube-dl', async function() { + beforeEach(async function () { + if (fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.unlinkSync(CONSTS.DETAILS_BIN_PATH); + await youtubedl_api.checkForYoutubeDLUpdate(); + }); + it('Check latest version', async function() { + this.timeout(300000); + const original_fork = config_api.getConfigItem('ytdl_default_downloader'); + const latest_version = await youtubedl_api.getLatestUpdateVersion(original_fork); + assert(latest_version > CONSTS.OUTDATED_YOUTUBEDL_VERSION); + }); + + it('Update youtube-dl', async function() { + this.timeout(300000); + const original_fork = config_api.getConfigItem('ytdl_default_downloader'); + const binary_path = path.join('test', 'test_binary'); + for (const youtubedl_fork in youtubedl_api.youtubedl_forks) { + config_api.setConfigItem('ytdl_default_downloader', youtubedl_fork); + const latest_version = await youtubedl_api.checkForYoutubeDLUpdate(); + await youtubedl_api.updateYoutubeDL(latest_version, binary_path); + assert(fs.existsSync(binary_path)); + if (fs.existsSync(binary_path)) fs.unlinkSync(binary_path); + } + config_api.setConfigItem('ytdl_default_downloader', original_fork); + }); + + it('Run process', async function() { + this.timeout(300000); + const downloader_api = require('../downloader'); + const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI'; + const args = await downloader_api.generateArgs(url, 'video', {}, null, true); + const {child_process} = await youtubedl_api.runYoutubeDL(url, args); + assert(child_process); + }); +}); + +describe('Subscriptions', function() { + const new_sub = { + name: 'test_sub', + url: 'https://www.youtube.com/channel/UCzofo-P8yMMCOv8rsPfIR-g', + maxQuality: null, + id: uuid(), + user_uid: null, + type: 'video', + paused: true + }; + beforeEach(async function() { + await db_api.removeAllRecords('subscriptions'); + }); + it('Subscribe', async function () { + const success = await subscriptions_api.subscribe(new_sub, null, true); + assert(success); + const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(sub_exists); + }); + it('Unsubscribe', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + await subscriptions_api.unsubscribe(new_sub); + const sub_exists = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(!sub_exists); + }); + it('Delete subscription file', async function () { + + }); + it('Get subscription by name', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + const sub_by_name = await subscriptions_api.getSubscriptionByName('test_sub'); + assert(sub_by_name); + }); + it('Get subscriptions', async function() { + await subscriptions_api.subscribe(new_sub, null, true); + const subs = await subscriptions_api.getSubscriptions(null); + assert(subs && subs.length === 1); + }); + it('Update subscription', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + const sub_update = Object.assign({}, new_sub, {name: 'updated_name'}); + await subscriptions_api.updateSubscription(sub_update); + const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(updated_sub['name'] === 'updated_name'); + }); + it('Update subscription property', async function () { + await subscriptions_api.subscribe(new_sub, null, true); + const sub_update = Object.assign({}, new_sub, {name: 'updated_name'}); + await subscriptions_api.updateSubscriptionPropertyMultiple([sub_update], {name: 'updated_name'}); + const updated_sub = await db_api.getRecord('subscriptions', {id: new_sub['id']}); + assert(updated_sub['name'] === 'updated_name'); + }); + it('Write subscription metadata', async function() { + const metadata_path = path.join('subscriptions', 'channels', 'test_sub', 'subscription_backup.json'); + if (fs.existsSync(metadata_path)) fs.unlinkSync(metadata_path); + await subscriptions_api.subscribe(new_sub, null, true); + assert(fs.existsSync(metadata_path)); + }); + it('Fresh uploads', async function() { + + }); +}); + describe('Tasks', function() { const tasks_api = require('../tasks'); beforeEach(async function() { @@ -635,7 +785,7 @@ describe('Tasks', function() { const success = await generateEmptyVideoFile('test/sample_mp4.mp4'); // pre-test cleanup - await db_api.removeAllRecords('files', {title: 'Sample File'}); + await db_api.removeAllRecords('files', {path: 'test/missing_file.mp4'}); if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json'); if (fs.existsSync('video/sample_mp4.mp4')) fs.unlinkSync('video/sample_mp4.mp4'); @@ -792,7 +942,7 @@ describe('Categories', async function() { rules: [], custom_output: '' }; - + await db_api.removeAllRecords('categories', {name: 'test_category'}); await db_api.insertRecordIntoTable('categories', new_category); }); diff --git a/backend/utils.js b/backend/utils.js index 3c7b794..27d0f66 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -241,11 +241,6 @@ exports.addUIDsToCategory = (category, files) => { return files_that_match; } -exports.getCurrentDownloader = () => { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - return details_json['downloader']; -} - exports.recFindByExt = async (base, ext, files, result, recursive = true) => { files = files || (await fs.readdir(base)) result = result || [] diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js index c951589..e48c292 100644 --- a/backend/youtube-dl.js +++ b/backend/youtube-dl.js @@ -1,151 +1,159 @@ const fs = require('fs-extra'); const fetch = require('node-fetch'); +const path = require('path'); +const execa = require('execa'); +const kill = require('tree-kill'); const logger = require('./logger'); const utils = require('./utils'); const CONSTS = require('./consts'); const config_api = require('./config.js'); -const youtubedl = require('youtube-dl'); - -const OUTDATED_VERSION = "2020.00.00"; const is_windows = process.platform === 'win32'; -const download_sources = { +exports.youtubedl_forks = { 'youtube-dl': { - 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', - 'func': downloadLatestYoutubeDLBinary + 'download_url': 'https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl', + 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags' }, 'youtube-dlc': { - 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', - 'func': downloadLatestYoutubeDLCBinary + 'download_url': 'https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc', + 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags' }, 'yt-dlp': { - 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', - 'func': downloadLatestYoutubeDLPBinary + 'download_url': 'https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp', + 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags' } } -exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => { +exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => { + const output_file_path = getYoutubeDLPath(); + if (!fs.existsSync(output_file_path)) await exports.checkForYoutubeDLUpdate(); + let callback = null; + let child_process = null; + if (customDownloadHandler) { + callback = runYoutubeDLCustom(url, args, customDownloadHandler); + } else { + ({callback, child_process} = await runYoutubeDLProcess(url, args)); + } + + return {child_process, callback}; +} + +// Run youtube-dl directly (not cancellable) +const runYoutubeDLCustom = async (url, args, customDownloadHandler) => { + const downloadHandler = customDownloadHandler; return new Promise(resolve => { - downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) { + downloadHandler(url, args, {maxBuffer: Infinity}, async function(err, output) { const parsed_output = utils.parseOutputJSON(output, err); resolve({parsed_output, err}); }); }); } -exports.checkForYoutubeDLUpdate = async () => { - return new Promise(async resolve => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); - const tags_url = download_sources[default_downloader]['tags_url']; - // get current version - let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); - if (!current_app_details_exists) { - logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader}); - } - let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); - let current_version = current_app_details['version']; - let current_downloader = current_app_details['downloader']; - let stored_binary_path = current_app_details['path']; - if (!stored_binary_path || typeof stored_binary_path !== 'string') { - // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); - const guessed_base_path = 'node_modules/youtube-dl/bin/'; - const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); - if (fs.existsSync(guessed_file_path)) { - stored_binary_path = guessed_file_path; - // logger.info('INFO: Guess successful! Update process continuing...') - } else { - logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); - resolve(null); - return; - } +// Run youtube-dl in a subprocess (cancellable) +const runYoutubeDLProcess = async (url, args, youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) => { + const youtubedl_path = getYoutubeDLPath(youtubedl_fork); + const binary_exists = fs.existsSync(youtubedl_path); + if (!binary_exists) { + const err = `Could not find path for ${youtubedl_fork} at ${youtubedl_path}`; + logger.error(err); + return; + } + const child_process = execa(getYoutubeDLPath(youtubedl_fork), [url, ...args], {maxBuffer: Infinity}); + const callback = new Promise(async resolve => { + try { + const {stdout, stderr} = await child_process; + const parsed_output = utils.parseOutputJSON(stdout.trim().split(/\r?\n/), stderr); + resolve({parsed_output, err: stderr}); + } catch (e) { + resolve({parsed_output: null, err: e}) } + }); + return {child_process, callback} +} + +function getYoutubeDLPath(youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) { + const binary_file_name = youtubedl_fork + (is_windows ? '.exe' : ''); + const binary_path = path.join('appdata', 'bin', binary_file_name); + return binary_path; +} + +exports.killYoutubeDLProcess = async (child_process) => { + kill(child_process.pid, 'SIGKILL'); +} + +exports.checkForYoutubeDLUpdate = async () => { + const selected_fork = config_api.getConfigItem('ytdl_default_downloader'); + const output_file_path = getYoutubeDLPath(); + // get current version + let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); + if (!current_app_details_exists) { + logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); + updateDetailsJSON(CONSTS.OUTDATED_YOUTUBEDL_VERSION, selected_fork, output_file_path); + } + const current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); + const current_version = current_app_details['version']; + const current_fork = current_app_details['downloader']; + + const latest_version = await exports.getLatestUpdateVersion(selected_fork); + // if the binary does not exist, or default_downloader doesn't match existing fork, or if the fork has been updated, redownload + // TODO: don't redownload if fork already exists + if (!fs.existsSync(output_file_path) || current_fork !== selected_fork || !current_version || current_version !== latest_version) { + logger.warn(`Updating ${selected_fork} binary to '${output_file_path}', downloading...`); + await exports.updateYoutubeDL(latest_version); + } +} + +exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => { + await fs.ensureDir(path.join('appdata', 'bin')); + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path); +} + +async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, custom_output_path = null) { + const file_ext = is_windows ? '.exe' : ''; + + // build the URL + const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`; + const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork); - // got version, now let's check the latest version from the youtube-dl API + await utils.fetchFile(download_url, output_path, `${youtubedl_fork} ${new_version}`); + fs.chmod(output_path, 0o777); + updateDetailsJSON(new_version, youtubedl_fork, output_path); +} +exports.getLatestUpdateVersion = async (youtubedl_fork) => { + const tags_url = exports.youtubedl_forks[youtubedl_fork]['tags_url']; + return new Promise(resolve => { fetch(tags_url, {method: 'Get'}) .then(async res => res.json()) .then(async (json) => { - // check if the versions are different if (!json || !json[0]) { - logger.error(`Failed to check ${default_downloader} version for an update.`) + logger.error(`Failed to check ${youtubedl_fork} version for an update.`) resolve(null); return; } const latest_update_version = json[0]['name']; - if (current_version !== latest_update_version || default_downloader !== current_downloader) { - // versions different or different downloader is being used, download new update - resolve(latest_update_version); - } else { - resolve(null); - } - return; + resolve(latest_update_version); }) .catch(err => { - logger.error(`Failed to check ${default_downloader} version for an update.`) + logger.error(`Failed to check ${youtubedl_fork} version for an update.`) logger.error(err); resolve(null); - return; }); }); } -exports.updateYoutubeDL = async (latest_update_version) => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); - await download_sources[default_downloader]['func'](latest_update_version); -} - -exports.verifyBinaryExistsLinux = () => { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) { - details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl'; - details_json['exec'] = 'youtube-dl'; - details_json['version'] = OUTDATED_VERSION; - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); - - utils.restartServer(); - } -} - -async function downloadLatestYoutubeDLBinary(new_version) { +function updateDetailsJSON(new_version, fork, output_path) { const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dl'); -} - -async function downloadLatestYoutubeDLCBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dlc'); -} - -async function downloadLatestYoutubeDLPBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`); - - updateDetailsJSON(new_version, 'yt-dlp'); -} - -function updateDetailsJSON(new_version, downloader) { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (new_version) details_json['version'] = new_version; - details_json['downloader'] = downloader; + const details_json = fs.existsSync(CONSTS.DETAILS_BIN_PATH) ? fs.readJSONSync(CONSTS.DETAILS_BIN_PATH) : {}; + if (!details_json[fork]) details_json[fork] = {}; + const fork_json = details_json[fork]; + fork_json['version'] = new_version; + fork_json['downloader'] = fork; + fork_json['path'] = output_path; // unused + fork_json['exec'] = fork + file_ext; // unused fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); } diff --git a/src/api-types/index.ts b/src/api-types/index.ts index c749b0b..04aa395 100644 --- a/src/api-types/index.ts +++ b/src/api-types/index.ts @@ -14,6 +14,7 @@ export type { ChangeRolePermissionsRequest } from './models/ChangeRolePermission export type { ChangeUserPermissionsRequest } from './models/ChangeUserPermissionsRequest'; export type { CheckConcurrentStreamRequest } from './models/CheckConcurrentStreamRequest'; export type { CheckConcurrentStreamResponse } from './models/CheckConcurrentStreamResponse'; +export type { CheckSubscriptionRequest } from './models/CheckSubscriptionRequest'; export type { ClearDownloadsRequest } from './models/ClearDownloadsRequest'; export type { ConcurrentStream } from './models/ConcurrentStream'; export type { Config } from './models/Config'; diff --git a/src/api-types/models/CheckSubscriptionRequest.ts b/src/api-types/models/CheckSubscriptionRequest.ts new file mode 100644 index 0000000..2d1aaeb --- /dev/null +++ b/src/api-types/models/CheckSubscriptionRequest.ts @@ -0,0 +1,7 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type CheckSubscriptionRequest = { + sub_id: string; +}; diff --git a/src/api-types/models/Download.ts b/src/api-types/models/Download.ts index 84d95e5..52c3529 100644 --- a/src/api-types/models/Download.ts +++ b/src/api-types/models/Download.ts @@ -8,6 +8,7 @@ export type Download = { running: boolean; finished: boolean; paused: boolean; + cancelled?: boolean; finished_step: boolean; url: string; type: string; diff --git a/src/api-types/models/Subscription.ts b/src/api-types/models/Subscription.ts index 9d84e2c..e54947f 100644 --- a/src/api-types/models/Subscription.ts +++ b/src/api-types/models/Subscription.ts @@ -11,9 +11,12 @@ export type Subscription = { type: FileType; user_uid: string | null; isPlaylist: boolean; + child_process?: any; archive?: string; timerange?: string; custom_args?: string; custom_output?: string; + downloading?: boolean; + paused?: boolean; videos: Array; }; diff --git a/src/api-types/models/UnsubscribeRequest.ts b/src/api-types/models/UnsubscribeRequest.ts index 9eed3c5..09692c1 100644 --- a/src/api-types/models/UnsubscribeRequest.ts +++ b/src/api-types/models/UnsubscribeRequest.ts @@ -2,10 +2,8 @@ /* tslint:disable */ /* eslint-disable */ -import type { SubscriptionRequestData } from './SubscriptionRequestData'; - export type UnsubscribeRequest = { - sub: SubscriptionRequestData; + sub_id: string; /** * Defaults to false */ diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index d5e3092..9490e8d 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -69,8 +69,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { tooltip: $localize`Pause`, action: (download: Download) => this.pauseDownload(download), show: (download: Download) => !download.finished && (!download.paused || !download.finished_step), - icon: 'pause', - loading: (download: Download) => download.paused && !download.finished_step + icon: 'pause' }, { tooltip: $localize`Resume`, @@ -81,7 +80,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { { tooltip: $localize`Cancel`, action: (download: Download) => this.cancelDownload(download), - show: (download: Download) => false && !download.finished && !download.paused, // TODO: add possibility to cancel download + show: (download: Download) => !download.finished && !download.paused && !download.cancelled, icon: 'cancel' }, { diff --git a/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.ts b/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.ts index 5e14fc3..0a7783e 100644 --- a/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.ts +++ b/src/app/dialogs/subscription-info-dialog/subscription-info-dialog.component.ts @@ -2,6 +2,7 @@ import { Component, OnInit, Inject } from '@angular/core'; import { MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; import { PostsService } from 'app/posts.services'; import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.component'; +import { Subscription } from 'api-types'; @Component({ selector: 'app-subscription-info-dialog', @@ -10,7 +11,7 @@ import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.compone }) export class SubscriptionInfoDialogComponent implements OnInit { - sub = null; + sub: Subscription = null; unsubbedEmitter = null; constructor(public dialogRef: MatDialogRef, @@ -41,7 +42,7 @@ export class SubscriptionInfoDialogComponent implements OnInit { } unsubscribe() { - this.postsService.unsubscribe(this.sub, true).subscribe(res => { + this.postsService.unsubscribe(this.sub.id, true).subscribe(res => { this.unsubbedEmitter.emit(true); this.dialogRef.close(); }); diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index 731dbe6..3a4058b 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -169,6 +169,8 @@ export class MainComponent implements OnInit { argsChangedSubject: Subject = new Subject(); simulatedOutput = ''; + interval_id = null; + constructor(public postsService: PostsService, private youtubeSearch: YoutubeSearchService, public snackBar: MatSnackBar, private router: Router, public dialog: MatDialog, private platform: Platform, private route: ActivatedRoute) { this.audioOnly = false; @@ -232,11 +234,12 @@ export class MainComponent implements OnInit { } // get downloads routine - setInterval(() => { + if (this.interval_id) { clearInterval(this.interval_id) } + this.interval_id = setInterval(() => { if (this.current_download) { this.getCurrentDownload(); } - }, 500); + }, 1000); return true; } @@ -294,6 +297,10 @@ export class MainComponent implements OnInit { } } + ngOnDestroy(): void { + if (this.interval_id) { clearInterval(this.interval_id) } + } + // download helpers downloadHelper(container: DatabaseFile | Playlist, type: string, is_playlist = false, force_view = false, navigate_mode = false): void { this.downloadingfile = false; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 67f86e3..e1a4af1 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -113,7 +113,8 @@ import { Archive, Subscription, RestartDownloadResponse, - TaskType + TaskType, + CheckSubscriptionRequest } from '../api-types'; import { isoLangs } from './dialogs/user-profile-dialog/locales_list'; import { Title } from '@angular/platform-browser'; @@ -566,8 +567,18 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'updateSubscription', {subscription: subscription}, this.httpOptions); } - unsubscribe(sub: SubscriptionRequestData, deleteMode = false) { - const body: UnsubscribeRequest = {sub: sub, deleteMode: deleteMode}; + checkSubscription(sub_id: string) { + const body: CheckSubscriptionRequest = {sub_id: sub_id}; + return this.http.post(this.path + 'checkSubscription', body, this.httpOptions); + } + + cancelCheckSubscription(sub_id: string) { + const body: CheckSubscriptionRequest = {sub_id: sub_id}; + return this.http.post(this.path + 'cancelCheckSubscription', body, this.httpOptions); + } + + unsubscribe(sub_id: string, deleteMode = false) { + const body: UnsubscribeRequest = {sub_id: sub_id, deleteMode: deleteMode}; return this.http.post(this.path + 'unsubscribe', body, this.httpOptions) } diff --git a/src/app/subscription/subscription/subscription.component.html b/src/app/subscription/subscription/subscription.component.html index 661279a..9384545 100644 --- a/src/app/subscription/subscription/subscription.component.html +++ b/src/app/subscription/subscription/subscription.component.html @@ -3,6 +3,7 @@

{{subscription.name}} (Paused) +

@@ -13,7 +14,14 @@
- +
+ + + + + + +
\ No newline at end of file diff --git a/src/app/subscription/subscription/subscription.component.scss b/src/app/subscription/subscription/subscription.component.scss index d82a706..7dd757e 100644 --- a/src/app/subscription/subscription/subscription.component.scss +++ b/src/app/subscription/subscription/subscription.component.scss @@ -58,13 +58,19 @@ bottom: 25px; } -.edit-button { +.check-button { left: 25px; position: fixed; bottom: 25px; z-index: 99999; } +.edit-button { + right: 35px; + position: fixed; + z-index: 99999; +} + .save-icon { bottom: 1px; position: relative; diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index 4c73efc..8b98cea 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -3,6 +3,7 @@ import { PostsService } from 'app/posts.services'; import { ActivatedRoute, Router, ParamMap } from '@angular/router'; import { MatDialog } from '@angular/material/dialog'; import { EditSubscriptionDialogComponent } from 'app/dialogs/edit-subscription-dialog/edit-subscription-dialog.component'; +import { Subscription } from 'api-types'; @Component({ selector: 'app-subscription', @@ -12,11 +13,13 @@ import { EditSubscriptionDialogComponent } from 'app/dialogs/edit-subscription-d export class SubscriptionComponent implements OnInit, OnDestroy { id = null; - subscription = null; + subscription: Subscription = null; use_youtubedl_archive = false; descendingMode = true; downloading = false; sub_interval = null; + check_clicked = false; + cancel_clicked = false; constructor(private postsService: PostsService, private route: ActivatedRoute, private router: Router, private dialog: MatDialog) { } @@ -90,4 +93,34 @@ export class SubscriptionComponent implements OnInit, OnDestroy { this.router.navigate(['/player', {sub_id: this.subscription.id}]) } + checkSubscription(): void { + this.check_clicked = true; + this.postsService.checkSubscription(this.subscription.id).subscribe(res => { + this.check_clicked = false; + if (!res['success']) { + this.postsService.openSnackBar('Failed to check subscription!'); + return; + } + }, err => { + console.error(err); + this.check_clicked = false; + this.postsService.openSnackBar('Failed to check subscription!'); + }); + } + + cancelCheckSubscription(): void { + this.cancel_clicked = true; + this.postsService.cancelCheckSubscription(this.subscription.id).subscribe(res => { + this.cancel_clicked = false; + if (!res['success']) { + this.postsService.openSnackBar('Failed to cancel check subscription!'); + return; + } + }, err => { + console.error(err); + this.cancel_clicked = false; + this.postsService.openSnackBar('Failed to cancel check subscription!'); + }); + } + }