You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
599 lines
24 KiB
JavaScript
599 lines
24 KiB
JavaScript
const fs = require('fs-extra');
|
|
const path = require('path');
|
|
|
|
const youtubedl_api = require('./youtube-dl');
|
|
const config_api = require('./config');
|
|
const archive_api = require('./archive');
|
|
const utils = require('./utils');
|
|
const logger = require('./logger');
|
|
const CONSTS = require('./consts');
|
|
|
|
const debugMode = process.env.YTDL_MODE === 'debug';
|
|
|
|
const db_api = require('./db');
|
|
const downloader_api = require('./downloader');
|
|
|
|
exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => {
|
|
const result_obj = {
|
|
success: false,
|
|
error: ''
|
|
};
|
|
return new Promise(async resolve => {
|
|
// sub should just have url and name. here we will get isPlaylist and path
|
|
sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist');
|
|
sub.videos = [];
|
|
|
|
let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));
|
|
|
|
if (!sub.name && url_exists) {
|
|
logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
|
|
result_obj.error = 'Subcription with URL ' + sub.url + ' already exists! Custom name is required.';
|
|
resolve(result_obj);
|
|
return;
|
|
}
|
|
|
|
sub['user_uid'] = user_uid ? user_uid : undefined;
|
|
await db_api.insertRecordIntoTable('subscriptions', JSON.parse(JSON.stringify(sub)));
|
|
|
|
let success = skip_get_info ? true : await getSubscriptionInfo(sub);
|
|
exports.writeSubscriptionMetadata(sub);
|
|
|
|
if (success) {
|
|
if (!sub.paused) exports.getVideosForSub(sub.id);
|
|
} else {
|
|
logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
|
|
}
|
|
|
|
result_obj.success = success;
|
|
result_obj.sub = sub;
|
|
resolve(result_obj);
|
|
});
|
|
|
|
}
|
|
|
|
async function getSubscriptionInfo(sub) {
|
|
// get videos
|
|
let downloadConfig = ['--dump-json', '--playlist-end', '1'];
|
|
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
|
|
if (useCookies) {
|
|
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
|
|
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
|
|
} else {
|
|
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
|
|
}
|
|
}
|
|
|
|
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
|
|
const {parsed_output, err} = await callback;
|
|
if (err) {
|
|
logger.error(err.stderr);
|
|
return false;
|
|
}
|
|
logger.verbose('Subscribe: got info for subscription ' + sub.id);
|
|
for (const output_json of parsed_output) {
|
|
if (!output_json) {
|
|
continue;
|
|
}
|
|
|
|
if (!sub.name) {
|
|
if (sub.isPlaylist) {
|
|
sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
|
|
} else {
|
|
sub.name = output_json.uploader;
|
|
}
|
|
// if it's now valid, update
|
|
if (sub.name) {
|
|
let sub_name = sub.name;
|
|
const sub_name_exists = await db_api.getRecord('subscriptions', {name: sub.name, isPlaylist: sub.isPlaylist, user_uid: sub.user_uid});
|
|
if (sub_name_exists) sub_name += ` - ${sub.id}`;
|
|
await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub_name});
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
exports.unsubscribe = async (sub_id, deleteMode, user_uid = null) => {
|
|
const sub = await exports.getSubscription(sub_id);
|
|
let basePath = null;
|
|
if (user_uid)
|
|
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
|
|
else
|
|
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
|
|
|
|
let id = sub.id;
|
|
|
|
const sub_files = await db_api.getRecords('files', {sub_id: id});
|
|
for (let i = 0; i < sub_files.length; i++) {
|
|
const sub_file = sub_files[i];
|
|
if (config_api.descriptors[sub_file['uid']]) {
|
|
try {
|
|
for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
|
|
config_api.descriptors[sub_file['uid']][i].destroy();
|
|
}
|
|
} catch(e) {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
await killSubDownloads(sub_id, true);
|
|
await db_api.removeRecord('subscriptions', {id: id});
|
|
await db_api.removeAllRecords('files', {sub_id: id});
|
|
|
|
// failed subs have no name, on unsubscribe they shouldn't error
|
|
if (!sub.name) {
|
|
return;
|
|
}
|
|
|
|
const appendedBasePath = getAppendedBasePath(sub, basePath);
|
|
if (deleteMode && (await fs.pathExists(appendedBasePath))) {
|
|
await fs.remove(appendedBasePath);
|
|
}
|
|
|
|
await db_api.removeAllRecords('archives', {sub_id: sub.id});
|
|
}
|
|
|
|
exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => {
|
|
if (typeof sub === 'string') {
|
|
// TODO: fix bad workaround where sub is a sub_id
|
|
sub = await db_api.getRecord('subscriptions', {sub_id: sub});
|
|
}
|
|
// TODO: combine this with deletefile
|
|
let basePath = null;
|
|
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
|
|
: config_api.getConfigItem('ytdl_subscriptions_base_path');
|
|
const appendedBasePath = getAppendedBasePath(sub, basePath);
|
|
const name = file;
|
|
let retrievedID = null;
|
|
let retrievedExtractor = null;
|
|
|
|
await db_api.removeRecord('files', {uid: file_uid});
|
|
|
|
let filePath = appendedBasePath;
|
|
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
|
|
var jsonPath = path.join(__dirname,filePath,name+'.info.json');
|
|
var videoFilePath = path.join(__dirname,filePath,name+ext);
|
|
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
|
|
var altImageFilePath = path.join(__dirname,filePath,name+'.webp');
|
|
|
|
const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
|
|
fs.pathExists(jsonPath),
|
|
fs.pathExists(videoFilePath),
|
|
fs.pathExists(imageFilePath),
|
|
fs.pathExists(altImageFilePath),
|
|
]);
|
|
|
|
if (jsonExists) {
|
|
const info_json = fs.readJSONSync(jsonPath);
|
|
retrievedID = info_json['id'];
|
|
retrievedExtractor = info_json['extractor'];
|
|
await fs.unlink(jsonPath);
|
|
}
|
|
|
|
if (imageFileExists) {
|
|
await fs.unlink(imageFilePath);
|
|
}
|
|
|
|
if (altImageFileExists) {
|
|
await fs.unlink(altImageFilePath);
|
|
}
|
|
|
|
if (videoFileExists) {
|
|
await fs.unlink(videoFilePath);
|
|
if ((await fs.pathExists(jsonPath)) || (await fs.pathExists(videoFilePath))) {
|
|
return false;
|
|
} else {
|
|
// check if the user wants the video to be redownloaded (deleteForever === false)
|
|
if (deleteForever) {
|
|
// ensure video is in the archives
|
|
const exists_in_archive = await archive_api.existsInArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
|
|
if (!exists_in_archive) {
|
|
await archive_api.addToArchive(retrievedExtractor, retrievedID, sub.type, file.title, user_uid, sub.id);
|
|
}
|
|
} else {
|
|
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
|
|
}
|
|
return true;
|
|
}
|
|
} else {
|
|
// TODO: tell user that the file didn't exist
|
|
return true;
|
|
}
|
|
}
|
|
|
|
let current_sub_index = 0; // To keep track of the current subscription
|
|
exports.watchSubscriptionsInterval = async () => {
|
|
const subscriptions_check_interval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
|
|
let parent_interval = setInterval(() => watchSubscriptions(), subscriptions_check_interval*1000);
|
|
watchSubscriptions();
|
|
config_api.config_updated.subscribe(change => {
|
|
if (!change) return;
|
|
if (change['key'] === 'ytdl_subscriptions_check_interval' || change['key'] === 'ytdl_multi_user_mode') {
|
|
current_sub_index = 0; // TODO: start after the last sub check
|
|
logger.verbose('Resetting sub check schedule due to config change');
|
|
clearInterval(parent_interval);
|
|
const new_interval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
|
|
parent_interval = setInterval(() => watchSubscriptions(), new_interval*1000);
|
|
watchSubscriptions();
|
|
}
|
|
});
|
|
}
|
|
|
|
async function watchSubscriptions() {
|
|
const subscription_ids = await getValidSubscriptionsToCheck();
|
|
if (subscription_ids.length === 0) {
|
|
logger.info('Skipping subscription check as no valid subscriptions exist.');
|
|
return;
|
|
}
|
|
checkSubscription(subscription_ids[current_sub_index]);
|
|
current_sub_index = (current_sub_index + 1) % subscription_ids.length;
|
|
}
|
|
|
|
async function checkSubscription(sub_id) {
|
|
let sub = await exports.getSubscription(sub_id);
|
|
|
|
// don't check the sub if the last check for the same subscription has not completed
|
|
if (sub.downloading) {
|
|
logger.verbose(`Subscription: skipped checking ${sub.name} as it's downloading videos.`);
|
|
return;
|
|
}
|
|
|
|
if (!sub.name) {
|
|
logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`);
|
|
return;
|
|
}
|
|
|
|
await exports.getVideosForSub(sub.id);
|
|
}
|
|
|
|
async function getValidSubscriptionsToCheck() {
|
|
const subscriptions = await exports.getAllSubscriptions();
|
|
|
|
if (!subscriptions) return;
|
|
|
|
// auto pause deprecated streamingOnly mode
|
|
const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
|
|
exports.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
|
|
|
|
const valid_subscription_ids = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly).map(sub => sub.id);
|
|
return valid_subscription_ids;
|
|
}
|
|
|
|
exports.getVideosForSub = async (sub_id) => {
|
|
const sub = await exports.getSubscription(sub_id);
|
|
if (!sub || sub['downloading']) {
|
|
return false;
|
|
}
|
|
|
|
_getVideosForSub(sub);
|
|
return true;
|
|
}
|
|
|
|
async function _getVideosForSub(sub) {
|
|
const user_uid = sub['user_uid'];
|
|
updateSubscriptionProperty(sub, {downloading: true}, user_uid);
|
|
|
|
// get basePath
|
|
let basePath = null;
|
|
if (user_uid)
|
|
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
|
|
else
|
|
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
|
|
|
|
let appendedBasePath = getAppendedBasePath(sub, basePath);
|
|
fs.ensureDirSync(appendedBasePath);
|
|
|
|
const downloadConfig = await generateArgsForSubscription(sub, user_uid);
|
|
|
|
// get videos
|
|
logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
|
|
|
|
let {child_process, callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
|
|
updateSubscriptionProperty(sub, {child_process: child_process}, user_uid);
|
|
const {parsed_output, err} = await callback;
|
|
updateSubscriptionProperty(sub, {downloading: false, child_process: null}, user_uid);
|
|
if (!parsed_output) {
|
|
logger.error('Subscription check failed!');
|
|
if (err) logger.error(err);
|
|
return null;
|
|
}
|
|
|
|
// remove temporary archive file if it exists
|
|
const archive_path = path.join(appendedBasePath, 'archive.txt');
|
|
const archive_exists = await fs.pathExists(archive_path);
|
|
if (archive_exists) {
|
|
await fs.unlink(archive_path);
|
|
}
|
|
|
|
logger.verbose('Subscription: finished check for ' + sub.name);
|
|
const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
|
|
return files_to_download;
|
|
}
|
|
|
|
async function handleOutputJSON(output_jsons, sub, user_uid) {
|
|
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
|
|
await setFreshUploads(sub, user_uid);
|
|
checkVideosForFreshUploads(sub, user_uid);
|
|
}
|
|
|
|
if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) {
|
|
logger.verbose('No additional videos to download for ' + sub.name);
|
|
return [];
|
|
}
|
|
|
|
const files_to_download = await getFilesToDownload(sub, output_jsons);
|
|
const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
|
|
|
|
for (let j = 0; j < files_to_download.length; j++) {
|
|
const file_to_download = files_to_download[j];
|
|
file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']); // prevent download object from blowing up in size
|
|
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, [file_to_download]);
|
|
}
|
|
|
|
return files_to_download;
|
|
}
|
|
|
|
exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => {
|
|
let basePath = null;
|
|
if (user_uid)
|
|
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
|
|
else
|
|
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
|
|
|
|
let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
|
|
|
|
const base_download_options = {
|
|
maxHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
|
|
customFileFolderPath: getAppendedBasePath(sub, basePath),
|
|
customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
|
|
customArchivePath: path.join(basePath, 'archives', sub.name),
|
|
additionalArgs: sub.custom_args
|
|
}
|
|
|
|
return base_download_options;
|
|
}
|
|
|
|
async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
|
|
// get basePath
|
|
let basePath = null;
|
|
if (user_uid)
|
|
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
|
|
else
|
|
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
|
|
|
|
let appendedBasePath = getAppendedBasePath(sub, basePath);
|
|
|
|
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
|
|
|
|
let fullOutput = `"${appendedBasePath}/${file_output}.%(ext)s"`;
|
|
if (desired_path) {
|
|
fullOutput = `"${desired_path}.%(ext)s"`;
|
|
} else if (sub.custom_output) {
|
|
fullOutput = `"${appendedBasePath}/${sub.custom_output}.%(ext)s"`;
|
|
}
|
|
|
|
let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
|
|
|
|
let qualityPath = null;
|
|
if (sub.type && sub.type === 'audio') {
|
|
qualityPath = ['-f', 'bestaudio']
|
|
qualityPath.push('-x');
|
|
qualityPath.push('--audio-format', 'mp3');
|
|
} else {
|
|
if (!sub.maxQuality || sub.maxQuality === 'best') qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'];
|
|
else qualityPath = ['-f', `bestvideo[height<=${sub.maxQuality}]+bestaudio/best[height<=${sub.maxQuality}]`, '--merge-output-format', 'mp4'];
|
|
}
|
|
|
|
downloadConfig.push(...qualityPath)
|
|
|
|
// skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
|
|
const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
|
|
const archive_count = archive_text.split('\n').length - 1;
|
|
if (archive_count > 0) {
|
|
logger.verbose(`Generating temporary archive file for subscription ${sub.name} with ${archive_count} entries.`)
|
|
const archive_path = path.join(appendedBasePath, 'archive.txt');
|
|
await fs.writeFile(archive_path, archive_text);
|
|
downloadConfig.push('--download-archive', archive_path);
|
|
}
|
|
|
|
if (sub.custom_args) {
|
|
const customArgsArray = sub.custom_args.split(',,');
|
|
if (customArgsArray.indexOf('-f') !== -1) {
|
|
// if custom args has a custom quality, replce the original quality with that of custom args
|
|
const original_output_index = downloadConfig.indexOf('-f');
|
|
downloadConfig.splice(original_output_index, 2);
|
|
}
|
|
downloadConfig.push(...customArgsArray);
|
|
}
|
|
|
|
if (sub.timerange && !redownload) {
|
|
downloadConfig.push('--dateafter', sub.timerange);
|
|
}
|
|
|
|
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
|
|
if (useCookies) {
|
|
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
|
|
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
|
|
} else {
|
|
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
|
|
}
|
|
}
|
|
|
|
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
|
|
downloadConfig.push('--write-thumbnail');
|
|
}
|
|
|
|
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
|
|
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
|
|
downloadConfig.push('-r', rate_limit);
|
|
}
|
|
|
|
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
|
|
if (default_downloader === 'yt-dlp') {
|
|
downloadConfig.push('--no-clean-info-json');
|
|
}
|
|
|
|
downloadConfig = utils.filterArgs(downloadConfig, ['--write-comments']);
|
|
|
|
return downloadConfig;
|
|
}
|
|
|
|
async function getFilesToDownload(sub, output_jsons) {
|
|
const files_to_download = [];
|
|
for (let i = 0; i < output_jsons.length; i++) {
|
|
const output_json = output_jsons[i];
|
|
const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null, finished: false}));
|
|
if (file_missing) {
|
|
const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
|
|
if (file_with_path_exists) {
|
|
// or maybe just overwrite???
|
|
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
|
|
continue;
|
|
}
|
|
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
|
|
if (exists_in_archive) continue;
|
|
|
|
files_to_download.push(output_json);
|
|
}
|
|
}
|
|
return files_to_download;
|
|
}
|
|
|
|
exports.cancelCheckSubscription = async (sub_id) => {
|
|
const sub = await exports.getSubscription(sub_id);
|
|
if (!sub['downloading'] && !sub['child_process']) {
|
|
logger.error('Failed to cancel subscription check, verify that it is still running!');
|
|
return false;
|
|
}
|
|
|
|
// if check is ongoing
|
|
if (sub['child_process']) {
|
|
const child_process = sub['child_process'];
|
|
youtubedl_api.killYoutubeDLProcess(child_process);
|
|
}
|
|
|
|
// cancel activate video downloads
|
|
await killSubDownloads(sub_id);
|
|
|
|
return true;
|
|
}
|
|
|
|
async function killSubDownloads(sub_id, remove_downloads = false) {
|
|
const sub_downloads = await db_api.getRecords('download_queue', {sub_id: sub_id});
|
|
for (const sub_download of sub_downloads) {
|
|
if (sub_download['running'])
|
|
await downloader_api.cancelDownload(sub_download['uid']);
|
|
if (remove_downloads)
|
|
await db_api.removeRecord('download_queue', {uid: sub_download['uid']});
|
|
}
|
|
}
|
|
|
|
exports.getSubscriptions = async (user_uid = null) => {
|
|
// TODO: fix issue where the downloading property may not match getSubscription()
|
|
return await db_api.getRecords('subscriptions', {user_uid: user_uid});
|
|
}
|
|
|
|
exports.getAllSubscriptions = async () => {
|
|
const all_subs = await db_api.getRecords('subscriptions');
|
|
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
|
|
return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
|
|
}
|
|
|
|
exports.getSubscription = async (subID) => {
|
|
// stringify and parse because we may override the 'downloading' property
|
|
const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID})));
|
|
// now with the download_queue, we may need to override 'downloading'
|
|
const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true);
|
|
if (!sub['downloading']) sub['downloading'] = current_downloads > 0;
|
|
return sub;
|
|
}
|
|
|
|
exports.getSubscriptionByName = async (subName, user_uid = null) => {
|
|
return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
|
|
}
|
|
|
|
exports.updateSubscription = async (sub) => {
|
|
await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
|
|
exports.writeSubscriptionMetadata(sub);
|
|
return true;
|
|
}
|
|
|
|
exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => {
|
|
subs.forEach(async sub => {
|
|
await updateSubscriptionProperty(sub, assignment_obj);
|
|
});
|
|
}
|
|
|
|
async function updateSubscriptionProperty(sub, assignment_obj) {
|
|
// TODO: combine with updateSubscription
|
|
await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
|
|
return true;
|
|
}
|
|
|
|
exports.writeSubscriptionMetadata = (sub) => {
|
|
let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions')
|
|
: config_api.getConfigItem('ytdl_subscriptions_base_path');
|
|
const appendedBasePath = getAppendedBasePath(sub, basePath);
|
|
const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH);
|
|
|
|
fs.ensureDirSync(appendedBasePath);
|
|
fs.writeJSONSync(metadata_path, sub);
|
|
}
|
|
|
|
async function setFreshUploads(sub) {
|
|
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
|
|
if (!sub_files) return;
|
|
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
|
|
sub_files.forEach(async file => {
|
|
if (current_date === file['upload_date'].replace(/-/g, '')) {
|
|
// set upload as fresh
|
|
const file_uid = file['uid'];
|
|
await db_api.setVideoProperty(file_uid, {'fresh_upload': true});
|
|
}
|
|
});
|
|
}
|
|
|
|
async function checkVideosForFreshUploads(sub, user_uid) {
|
|
const sub_files = await db_api.getRecords('files', {sub_id: sub.id});
|
|
const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
|
|
sub_files.forEach(async file => {
|
|
if (file['fresh_upload'] && current_date > file['upload_date'].replace(/-/g, '')) {
|
|
await checkVideoIfBetterExists(file, sub, user_uid)
|
|
}
|
|
});
|
|
}
|
|
|
|
async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
|
|
const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4);
|
|
const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
|
|
logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
|
|
// simulate a download to verify that a better version exists
|
|
|
|
const info = await downloader_api.getVideoInfoByURL(file_obj['url'], downloadConfig);
|
|
if (info && info.length === 1) {
|
|
const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
|
|
if (info[metric_to_compare] > file_obj[metric_to_compare]) {
|
|
// download new video as the simulated one is better
|
|
let {callback} = await youtubedl_api.runYoutubeDL(sub.url, downloadConfig);
|
|
const {parsed_output, err} = await callback;
|
|
if (err) {
|
|
logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
|
|
} else if (parsed_output) {
|
|
logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${info[metric_to_compare]}`);
|
|
await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: info[metric_to_compare]});
|
|
}
|
|
}
|
|
}
|
|
await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false});
|
|
}
|
|
|
|
// helper functions
|
|
|
|
function getAppendedBasePath(sub, base_path) {
|
|
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
|
|
}
|