diff --git a/backend/app.js b/backend/app.js index 3eba4ec..beaaa11 100644 --- a/backend/app.js +++ b/backend/app.js @@ -61,7 +61,7 @@ config_api.initialize(); db_api.initialize(db, users_db); auth_api.initialize(db_api); downloader_api.initialize(db_api); -subscriptions_api.initialize(db_api); +subscriptions_api.initialize(db_api, downloader_api); categories_api.initialize(db_api); // Set some defaults @@ -533,6 +533,8 @@ async function loadConfig() { // connect to DB await db_api.connectToDB(); + db_api.database_initialized = true; + db_api.database_initialized_bs.next(true); // creates archive path if missing await fs.ensureDir(archivePath); diff --git a/backend/db.js b/backend/db.js index 4ea0bb2..a872b25 100644 --- a/backend/db.js +++ b/backend/db.js @@ -9,10 +9,13 @@ const logger = require('./logger'); const low = require('lowdb') const FileSync = require('lowdb/adapters/FileSync'); +const { BehaviorSubject } = require('rxjs'); const local_adapter = new FileSync('./appdata/local_db.json'); const local_db = low(local_adapter); let database = null; +exports.database_initialized = false; +exports.database_initialized_bs = new BehaviorSubject(false); const tables = { files: { diff --git a/backend/downloader.js b/backend/downloader.js index 3528168..99a34b8 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -26,16 +26,24 @@ function setDB(input_db_api) { db_api = input_db_api } exports.initialize = (input_db_api) => { setDB(input_db_api); categories_api.initialize(db_api); - setupDownloads(); + if (db_api.database_initialized) { + setupDownloads(); + } else { + db_api.database_initialized_bs.subscribe(init => { + if (init) setupDownloads(); + }); + } } -exports.createDownload = async (url, type, options, user_uid = null) => { +exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => { return await mutex.runExclusive(async () => { const download = { url: url, type: type, title: '', user_uid: user_uid, + sub_id: sub_id, + sub_name: sub_name, options: options, uid: uuid(), step_index: 0, @@ -116,7 +124,7 @@ async function setupDownloads() { async function fixDownloadState() { const downloads = await db_api.getRecords('download_queue'); downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start); - const running_downloads = downloads.filter(download => !download['finished_step']); + const running_downloads = downloads.filter(download => !download['finished'] && !download['error']); for (let i = 0; i < running_downloads.length; i++) { const running_download = running_downloads[i]; const update_obj = {finished_step: true, paused: true}; @@ -143,7 +151,7 @@ async function checkDownloads() { return; }); - const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step']); + const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']); for (let i = 0; i < waiting_downloads.length; i++) { const running_download = waiting_downloads[i]; if (i === 5/*config_api.getConfigItem('ytdl_max_concurrent_downloads')*/) break; @@ -322,7 +330,7 @@ async function downloadQueuedFile(download_uid) { } // registers file in DB - const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, null, options.cropFileSettings); + const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); file_objs.push(file_obj); } @@ -437,13 +445,20 @@ async function generateArgs(url, type, options, user_uid = null) { let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); if (useYoutubeDLArchive) { - const archive_folder = user_uid ? path.join(fileFolderPath, 'archives') : archivePath; + let archive_folder = null; + if (options.customArchivePath) { + archive_folder = path.join(options.customArchivePath); + } else if (user_uid) { + archive_folder = path.join(fileFolderPath, 'archives'); + } else { + archive_folder = path.join(archivePath); + } const archive_path = path.join(archive_folder, `archive_${type}.txt`); await fs.ensureDir(archive_folder); await fs.ensureFile(archive_path); - let blacklist_path = user_uid ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`); + let blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`); await fs.ensureFile(blacklist_path); let merged_path = path.join(fileFolderPath, `merged_${type}.txt`); @@ -471,6 +486,10 @@ async function generateArgs(url, type, options, user_uid = null) { downloadConfig = downloadConfig.concat(globalArgs.split(',,')); } + if (options.additionalArgs && options.additionalArgs !== '') { + downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,')); + } + const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit'); if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) { downloadConfig.push('-r', rate_limit); diff --git a/backend/subscriptions.js b/backend/subscriptions.js index ece25f6..3e31ea8 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -10,11 +10,13 @@ const logger = require('./logger'); const debugMode = process.env.YTDL_MODE === 'debug'; let db_api = null; +let downloader_api = null; function setDB(input_db_api) { db_api = input_db_api } -function initialize(input_db_api) { +function initialize(input_db_api, input_downloader_api) { setDB(input_db_api); + downloader_api = input_downloader_api; } async function subscribe(sub, user_uid = null) { @@ -107,22 +109,6 @@ async function getSubscriptionInfo(sub, user_uid = null) { } } - const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useArchive && !sub.archive) { - // must create the archive - const archive_dir = path.join(__dirname, basePath, 'archives', sub.name); - const archive_path = path.join(archive_dir, 'archive.txt'); - - // creates archive directory and text file if it doesn't exist - fs.ensureDirSync(archive_dir); - fs.ensureFileSync(archive_path); - - // updates subscription - sub.archive = archive_dir; - - await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir}); - } - // TODO: get even more info resolve(true); @@ -256,30 +242,15 @@ async function getVideosForSub(sub, user_uid = null) { let appendedBasePath = getAppendedBasePath(sub, basePath); fs.ensureDirSync(appendedBasePath); - let multiUserMode = null; - if (user_uid) { - multiUserMode = { - user: user_uid, - file_path: appendedBasePath - } - } - const downloadConfig = await generateArgsForSubscription(sub, user_uid); // get videos logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`); return new Promise(async resolve => { - const preimported_file_paths = []; - const PREIMPORT_INTERVAL = 5000; - const preregister_check = setInterval(async () => { - if (sub.streamingOnly) return; - await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath); - }, PREIMPORT_INTERVAL); youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) { // cleanup updateSubscriptionProperty(sub, {downloading: false}, user_uid); - clearInterval(preregister_check); logger.verbose('Subscription: finished check for ' + sub.name); if (err && !output) { @@ -287,19 +258,21 @@ async function getVideosForSub(sub, user_uid = null) { if (err.stderr.includes('This video is unavailable')) { logger.info('An error was encountered with at least one video, backup method will be used.') try { - const outputs = err.stdout.split(/\r\n|\r|\n/); - for (let i = 0; i < outputs.length; i++) { - const output = JSON.parse(outputs[i]); - await handleOutputJSON(sub, output, i === 0, multiUserMode) - if (err.stderr.includes(output['id']) && archive_path) { - // we found a video that errored! add it to the archive to prevent future errors - if (sub.archive) { - archive_dir = sub.archive; - archive_path = path.join(archive_dir, 'archive.txt') - fs.appendFileSync(archive_path, output['id']); - } - } - } + // TODO: reimplement + + // const outputs = err.stdout.split(/\r\n|\r|\n/); + // for (let i = 0; i < outputs.length; i++) { + // const output = JSON.parse(outputs[i]); + // await handleOutputJSON(sub, output, i === 0, multiUserMode) + // if (err.stderr.includes(output['id']) && archive_path) { + // // we found a video that errored! add it to the archive to prevent future errors + // if (sub.archive) { + // archive_dir = sub.archive; + // archive_path = path.join(archive_dir, 'archive.txt') + // fs.appendFileSync(archive_path, output['id']); + // } + // } + // } } catch(e) { logger.error('Backup method failed. See error below:'); logger.error(e); @@ -312,21 +285,30 @@ async function getVideosForSub(sub, user_uid = null) { resolve(true); return; } + const output_jsons = []; for (let i = 0; i < output.length; i++) { let output_json = null; try { output_json = JSON.parse(output[i]); + output_jsons.push(output_json); } catch(e) { output_json = null; } if (!output_json) { continue; } + } - const reset_videos = i === 0; - await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos); + const files_to_download = await getFilesToDownload(sub, output_jsons); + const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid); + + for (let j = 0; j < files_to_download.length; j++) { + const file_to_download = files_to_download[j]; + await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name); } + resolve(files_to_download); + if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { await setFreshUploads(sub, user_uid); checkVideosForFreshUploads(sub, user_uid); @@ -338,10 +320,28 @@ async function getVideosForSub(sub, user_uid = null) { }, err => { logger.error(err); updateSubscriptionProperty(sub, {downloading: false}, user_uid); - clearInterval(preregister_check); }); } +function generateOptionsForSubscriptionDownload(sub, user_uid) { + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; + + const base_download_options = { + selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null, + customFileFolderPath: getAppendedBasePath(sub, basePath), + customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`, + customArchivePath: path.join(__dirname, basePath, 'archives', sub.name) + } + + return base_download_options; +} + async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) { // get basePath let basePath = null; @@ -363,7 +363,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`; } - let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json']; + let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json']; let qualityPath = null; if (sub.type && sub.type === 'audio') { @@ -378,7 +378,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push(...qualityPath) if (sub.custom_args) { - customArgsArray = sub.custom_args.split(',,'); + const customArgsArray = sub.custom_args.split(',,'); if (customArgsArray.indexOf('-f') !== -1) { // if custom args has a custom quality, replce the original quality with that of custom args const original_output_index = downloadConfig.indexOf('-f'); @@ -466,6 +466,24 @@ async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_vi } } +async function getFilesToDownload(sub, output_jsons) { + const files_to_download = []; + for (let i = 0; i < output_jsons.length; i++) { + const output_json = output_jsons[i]; + const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})); + if (file_missing) { + const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']}); + if (file_with_path_exists) { + // or maybe just overwrite??? + logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`) + } + files_to_download.push(output_json); + } + } + return files_to_download; +} + + async function getSubscriptions(user_uid = null) { return await db_api.getRecords('subscriptions', {user_uid: user_uid}); } diff --git a/src/app/components/downloads/downloads.component.html b/src/app/components/downloads/downloads.component.html index 50f2762..9ba56ec 100644 --- a/src/app/components/downloads/downloads.component.html +++ b/src/app/components/downloads/downloads.component.html @@ -17,6 +17,19 @@ + + + + Subscription + + + {{element.sub_name}} + + + N/A + + + @@ -41,15 +54,17 @@ Actions -
+
+ - -
-
+ + + - + +
diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index 380b77c..99bb028 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -52,7 +52,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { 3: 'Complete' } - displayedColumns: string[] = ['date', 'title', 'stage', 'progress', 'actions']; + displayedColumns: string[] = ['date', 'title', 'stage', 'subscription', 'progress', 'actions']; dataSource = null; // new MatTableDataSource(); downloads_retrieved = false; diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index fb4a5ce..cced686 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -246,11 +246,6 @@ export class MainComponent implements OnInit { this.useDefaultDownloadingAgent = this.postsService.config['Advanced']['use_default_downloading_agent']; this.customDownloadingAgent = this.postsService.config['Advanced']['custom_downloading_agent']; - if (this.youtubeSearchEnabled && this.youtubeAPIKey) { - this.youtubeSearch.initializeAPI(this.youtubeAPIKey); - this.attachToInput(); - } - // set final cache items localStorage.setItem('cached_filemanager_enabled', this.fileManagerEnabled.toString()); @@ -330,6 +325,13 @@ export class MainComponent implements OnInit { this.setCols(); } + ngAfterViewInit() { + if (this.youtubeSearchEnabled && this.youtubeAPIKey) { + this.youtubeSearch.initializeAPI(this.youtubeAPIKey); + this.attachToInput(); + } + } + public setCols() { if (window.innerWidth <= 350) { this.files_cols = 1; diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 40b02f9..dc87d3b 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -166,18 +166,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { const subscription = res['subscription']; this.subscription = subscription; this.type === this.subscription.type; - subscription.videos.forEach(video => { - if (video['uid'] === this.uid) { - this.db_file = video; - this.postsService.incrementViewCount(this.db_file['uid'], this.sub_id, this.uuid).subscribe(res => {}, err => { - console.error('Failed to increment view count'); - console.error(err); - }); - this.uids = [this.db_file['uid']]; - this.show_player = true; - this.parseFileNames(); - } - }); + this.uids = this.subscription.videos.map(video => video['uid']); + this.parseFileNames(); }, err => { this.openSnackBar(`Failed to find subscription ${this.sub_id}`, 'Dismiss'); }); @@ -205,7 +195,14 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { for (let i = 0; i < this.uids.length; i++) { const uid = this.uids[i]; - const file_obj = this.playlist_id ? this.db_playlist['file_objs'][i] : this.db_file; + let file_obj = null; + if (this.playlist_id) { + file_obj = this.db_playlist['file_objs'][i]; + } else if (this.sub_id) { + file_obj = this.subscription['videos'][i]; + } else { + file_obj = this.db_file; + } const mime_type = file_obj.isAudio ? 'audio/mp3' : 'video/mp4' diff --git a/src/app/subscription/subscription/subscription.component.html b/src/app/subscription/subscription/subscription.component.html index 82747ac..892108a 100644 --- a/src/app/subscription/subscription/subscription.component.html +++ b/src/app/subscription/subscription/subscription.component.html @@ -44,5 +44,6 @@
+ \ No newline at end of file diff --git a/src/app/subscription/subscription/subscription.component.scss b/src/app/subscription/subscription/subscription.component.scss index 562d3ab..373c448 100644 --- a/src/app/subscription/subscription/subscription.component.scss +++ b/src/app/subscription/subscription/subscription.component.scss @@ -67,4 +67,10 @@ .save-icon { bottom: 1px; position: relative; +} + +.watch-button { + left: 90px; + position: fixed; + bottom: 25px; } \ No newline at end of file diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index cb40b1c..e86789f 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -109,8 +109,7 @@ export class SubscriptionComponent implements OnInit, OnDestroy { if (this.subscription.streamingOnly) { this.router.navigate(['/player', {uid: uid, url: url}]); } else { - this.router.navigate(['/player', {uid: uid, - sub_id: this.subscription.id}]); + this.router.navigate(['/player', {uid: uid}]); } } @@ -171,4 +170,8 @@ export class SubscriptionComponent implements OnInit, OnDestroy { }); } + watchSubscription() { + this.router.navigate(['/player', {sub_id: this.subscription.id}]) + } + }