youtube-dl refactor (#956)
* Consolidated all youtube-dl calls into one function * Downloads can now be cancelled and better "paused" * Removed node-youtube-dl dependency * Added ability to manually check a subscription, and to cancel a subscription check --------- Co-authored-by: Dedy Martadinata S <dedyms@proton.me>pull/977/merge
Tzahi12345
1 year ago
committed by
GitHub
parent
99c5cf590e
commit
0565cf24a6
@ -1,151 +1,159 @@
|
|||||||
const fs = require('fs-extra');
|
const fs = require('fs-extra');
|
||||||
const fetch = require('node-fetch');
|
const fetch = require('node-fetch');
|
||||||
|
const path = require('path');
|
||||||
|
const execa = require('execa');
|
||||||
|
const kill = require('tree-kill');
|
||||||
|
|
||||||
const logger = require('./logger');
|
const logger = require('./logger');
|
||||||
const utils = require('./utils');
|
const utils = require('./utils');
|
||||||
const CONSTS = require('./consts');
|
const CONSTS = require('./consts');
|
||||||
const config_api = require('./config.js');
|
const config_api = require('./config.js');
|
||||||
const youtubedl = require('youtube-dl');
|
|
||||||
|
|
||||||
const OUTDATED_VERSION = "2020.00.00";
|
|
||||||
|
|
||||||
const is_windows = process.platform === 'win32';
|
const is_windows = process.platform === 'win32';
|
||||||
|
|
||||||
const download_sources = {
|
exports.youtubedl_forks = {
|
||||||
'youtube-dl': {
|
'youtube-dl': {
|
||||||
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
|
'download_url': 'https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl',
|
||||||
'func': downloadLatestYoutubeDLBinary
|
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags'
|
||||||
},
|
},
|
||||||
'youtube-dlc': {
|
'youtube-dlc': {
|
||||||
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
|
'download_url': 'https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc',
|
||||||
'func': downloadLatestYoutubeDLCBinary
|
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags'
|
||||||
},
|
},
|
||||||
'yt-dlp': {
|
'yt-dlp': {
|
||||||
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
|
'download_url': 'https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp',
|
||||||
'func': downloadLatestYoutubeDLPBinary
|
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
|
exports.runYoutubeDL = async (url, args, customDownloadHandler = null) => {
|
||||||
|
const output_file_path = getYoutubeDLPath();
|
||||||
|
if (!fs.existsSync(output_file_path)) await exports.checkForYoutubeDLUpdate();
|
||||||
|
let callback = null;
|
||||||
|
let child_process = null;
|
||||||
|
if (customDownloadHandler) {
|
||||||
|
callback = runYoutubeDLCustom(url, args, customDownloadHandler);
|
||||||
|
} else {
|
||||||
|
({callback, child_process} = await runYoutubeDLProcess(url, args));
|
||||||
|
}
|
||||||
|
|
||||||
|
return {child_process, callback};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run youtube-dl directly (not cancellable)
|
||||||
|
const runYoutubeDLCustom = async (url, args, customDownloadHandler) => {
|
||||||
|
const downloadHandler = customDownloadHandler;
|
||||||
return new Promise(resolve => {
|
return new Promise(resolve => {
|
||||||
downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
|
downloadHandler(url, args, {maxBuffer: Infinity}, async function(err, output) {
|
||||||
const parsed_output = utils.parseOutputJSON(output, err);
|
const parsed_output = utils.parseOutputJSON(output, err);
|
||||||
resolve({parsed_output, err});
|
resolve({parsed_output, err});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.checkForYoutubeDLUpdate = async () => {
|
// Run youtube-dl in a subprocess (cancellable)
|
||||||
return new Promise(async resolve => {
|
const runYoutubeDLProcess = async (url, args, youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) => {
|
||||||
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
|
const youtubedl_path = getYoutubeDLPath(youtubedl_fork);
|
||||||
const tags_url = download_sources[default_downloader]['tags_url'];
|
const binary_exists = fs.existsSync(youtubedl_path);
|
||||||
// get current version
|
if (!binary_exists) {
|
||||||
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
|
const err = `Could not find path for ${youtubedl_fork} at ${youtubedl_path}`;
|
||||||
if (!current_app_details_exists) {
|
logger.error(err);
|
||||||
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
|
return;
|
||||||
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader});
|
}
|
||||||
}
|
const child_process = execa(getYoutubeDLPath(youtubedl_fork), [url, ...args], {maxBuffer: Infinity});
|
||||||
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
|
const callback = new Promise(async resolve => {
|
||||||
let current_version = current_app_details['version'];
|
try {
|
||||||
let current_downloader = current_app_details['downloader'];
|
const {stdout, stderr} = await child_process;
|
||||||
let stored_binary_path = current_app_details['path'];
|
const parsed_output = utils.parseOutputJSON(stdout.trim().split(/\r?\n/), stderr);
|
||||||
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
|
resolve({parsed_output, err: stderr});
|
||||||
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
|
} catch (e) {
|
||||||
const guessed_base_path = 'node_modules/youtube-dl/bin/';
|
resolve({parsed_output: null, err: e})
|
||||||
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
|
|
||||||
if (fs.existsSync(guessed_file_path)) {
|
|
||||||
stored_binary_path = guessed_file_path;
|
|
||||||
// logger.info('INFO: Guess successful! Update process continuing...')
|
|
||||||
} else {
|
|
||||||
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
|
|
||||||
resolve(null);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
return {child_process, callback}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getYoutubeDLPath(youtubedl_fork = config_api.getConfigItem('ytdl_default_downloader')) {
|
||||||
|
const binary_file_name = youtubedl_fork + (is_windows ? '.exe' : '');
|
||||||
|
const binary_path = path.join('appdata', 'bin', binary_file_name);
|
||||||
|
return binary_path;
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.killYoutubeDLProcess = async (child_process) => {
|
||||||
|
kill(child_process.pid, 'SIGKILL');
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.checkForYoutubeDLUpdate = async () => {
|
||||||
|
const selected_fork = config_api.getConfigItem('ytdl_default_downloader');
|
||||||
|
const output_file_path = getYoutubeDLPath();
|
||||||
|
// get current version
|
||||||
|
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
|
||||||
|
if (!current_app_details_exists) {
|
||||||
|
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
|
||||||
|
updateDetailsJSON(CONSTS.OUTDATED_YOUTUBEDL_VERSION, selected_fork, output_file_path);
|
||||||
|
}
|
||||||
|
const current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
|
||||||
|
const current_version = current_app_details['version'];
|
||||||
|
const current_fork = current_app_details['downloader'];
|
||||||
|
|
||||||
|
const latest_version = await exports.getLatestUpdateVersion(selected_fork);
|
||||||
|
// if the binary does not exist, or default_downloader doesn't match existing fork, or if the fork has been updated, redownload
|
||||||
|
// TODO: don't redownload if fork already exists
|
||||||
|
if (!fs.existsSync(output_file_path) || current_fork !== selected_fork || !current_version || current_version !== latest_version) {
|
||||||
|
logger.warn(`Updating ${selected_fork} binary to '${output_file_path}', downloading...`);
|
||||||
|
await exports.updateYoutubeDL(latest_version);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.updateYoutubeDL = async (latest_update_version, custom_output_path = null) => {
|
||||||
|
await fs.ensureDir(path.join('appdata', 'bin'));
|
||||||
|
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
|
||||||
|
await downloadLatestYoutubeDLBinaryGeneric(default_downloader, latest_update_version, custom_output_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version, custom_output_path = null) {
|
||||||
|
const file_ext = is_windows ? '.exe' : '';
|
||||||
|
|
||||||
|
// build the URL
|
||||||
|
const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`;
|
||||||
|
const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork);
|
||||||
|
|
||||||
// got version, now let's check the latest version from the youtube-dl API
|
await utils.fetchFile(download_url, output_path, `${youtubedl_fork} ${new_version}`);
|
||||||
|
fs.chmod(output_path, 0o777);
|
||||||
|
|
||||||
|
updateDetailsJSON(new_version, youtubedl_fork, output_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.getLatestUpdateVersion = async (youtubedl_fork) => {
|
||||||
|
const tags_url = exports.youtubedl_forks[youtubedl_fork]['tags_url'];
|
||||||
|
return new Promise(resolve => {
|
||||||
fetch(tags_url, {method: 'Get'})
|
fetch(tags_url, {method: 'Get'})
|
||||||
.then(async res => res.json())
|
.then(async res => res.json())
|
||||||
.then(async (json) => {
|
.then(async (json) => {
|
||||||
// check if the versions are different
|
|
||||||
if (!json || !json[0]) {
|
if (!json || !json[0]) {
|
||||||
logger.error(`Failed to check ${default_downloader} version for an update.`)
|
logger.error(`Failed to check ${youtubedl_fork} version for an update.`)
|
||||||
resolve(null);
|
resolve(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const latest_update_version = json[0]['name'];
|
const latest_update_version = json[0]['name'];
|
||||||
if (current_version !== latest_update_version || default_downloader !== current_downloader) {
|
resolve(latest_update_version);
|
||||||
// versions different or different downloader is being used, download new update
|
|
||||||
resolve(latest_update_version);
|
|
||||||
} else {
|
|
||||||
resolve(null);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
})
|
})
|
||||||
.catch(err => {
|
.catch(err => {
|
||||||
logger.error(`Failed to check ${default_downloader} version for an update.`)
|
logger.error(`Failed to check ${youtubedl_fork} version for an update.`)
|
||||||
logger.error(err);
|
logger.error(err);
|
||||||
resolve(null);
|
resolve(null);
|
||||||
return;
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.updateYoutubeDL = async (latest_update_version) => {
|
function updateDetailsJSON(new_version, fork, output_path) {
|
||||||
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
|
|
||||||
await download_sources[default_downloader]['func'](latest_update_version);
|
|
||||||
}
|
|
||||||
|
|
||||||
exports.verifyBinaryExistsLinux = () => {
|
|
||||||
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
|
|
||||||
if (!is_windows && details_json && (!details_json['path'] || details_json['path'].includes('.exe'))) {
|
|
||||||
details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
|
|
||||||
details_json['exec'] = 'youtube-dl';
|
|
||||||
details_json['version'] = OUTDATED_VERSION;
|
|
||||||
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
|
|
||||||
|
|
||||||
utils.restartServer();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function downloadLatestYoutubeDLBinary(new_version) {
|
|
||||||
const file_ext = is_windows ? '.exe' : '';
|
const file_ext = is_windows ? '.exe' : '';
|
||||||
|
const details_json = fs.existsSync(CONSTS.DETAILS_BIN_PATH) ? fs.readJSONSync(CONSTS.DETAILS_BIN_PATH) : {};
|
||||||
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
|
if (!details_json[fork]) details_json[fork] = {};
|
||||||
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
|
const fork_json = details_json[fork];
|
||||||
|
fork_json['version'] = new_version;
|
||||||
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
|
fork_json['downloader'] = fork;
|
||||||
|
fork_json['path'] = output_path; // unused
|
||||||
updateDetailsJSON(new_version, 'youtube-dl');
|
fork_json['exec'] = fork + file_ext; // unused
|
||||||
}
|
|
||||||
|
|
||||||
async function downloadLatestYoutubeDLCBinary(new_version) {
|
|
||||||
const file_ext = is_windows ? '.exe' : '';
|
|
||||||
|
|
||||||
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
|
|
||||||
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
|
|
||||||
|
|
||||||
await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
|
|
||||||
|
|
||||||
updateDetailsJSON(new_version, 'youtube-dlc');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function downloadLatestYoutubeDLPBinary(new_version) {
|
|
||||||
const file_ext = is_windows ? '.exe' : '';
|
|
||||||
|
|
||||||
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
|
|
||||||
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
|
|
||||||
|
|
||||||
await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
|
|
||||||
|
|
||||||
updateDetailsJSON(new_version, 'yt-dlp');
|
|
||||||
}
|
|
||||||
|
|
||||||
function updateDetailsJSON(new_version, downloader) {
|
|
||||||
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
|
|
||||||
if (new_version) details_json['version'] = new_version;
|
|
||||||
details_json['downloader'] = downloader;
|
|
||||||
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
|
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,7 @@
|
|||||||
|
/* istanbul ignore file */
|
||||||
|
/* tslint:disable */
|
||||||
|
/* eslint-disable */
|
||||||
|
|
||||||
|
export type CheckSubscriptionRequest = {
|
||||||
|
sub_id: string;
|
||||||
|
};
|
||||||
Loading…
Reference in New Issue