From 09b3c752d9f00f2d4e451726bc444bca1919c878 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 24 Aug 2021 21:33:43 -0600 Subject: [PATCH] Removed downlload delay setting for subscriptions Subscription downloads already queued are now not requeued on the next check Headers in download queue table are now sortable Added button to clear all finished downloads in the downloads manager --- backend/appdata/default.json | 3 +- backend/config.js | 3 +- backend/downloader.js | 37 ++++++++++--------- backend/subscriptions.js | 11 +----- .../downloads/downloads.component.html | 17 +++++---- .../downloads/downloads.component.ts | 25 +++++++++++-- src/app/settings/settings.component.html | 6 --- 7 files changed, 54 insertions(+), 48 deletions(-) diff --git a/backend/appdata/default.json b/backend/appdata/default.json index c25bdce..3a26029 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -42,8 +42,7 @@ "allow_subscriptions": true, "subscriptions_base_path": "subscriptions/", "subscriptions_check_interval": "300", - "redownload_fresh_uploads": false, - "download_delay": "" + "redownload_fresh_uploads": false }, "Users": { "base_path": "users/", diff --git a/backend/config.js b/backend/config.js index 55469ef..fe710ac 100644 --- a/backend/config.js +++ b/backend/config.js @@ -217,8 +217,7 @@ const DEFAULT_CONFIG = { "allow_subscriptions": true, "subscriptions_base_path": "subscriptions/", "subscriptions_check_interval": "86400", - "redownload_fresh_uploads": false, - "download_delay": "" + "redownload_fresh_uploads": false }, "Users": { "base_path": "users/", diff --git a/backend/downloader.js b/backend/downloader.js index 99a34b8..dd07dab 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -48,6 +48,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu uid: uuid(), step_index: 0, paused: false, + running: false, finished_step: true, error: null, percent_complete: null, @@ -71,7 +72,7 @@ exports.pauseDownload = async (download_uid) => { return false; } - return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true}); + return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true, running: false}); } exports.resumeDownload = async (download_uid) => { @@ -106,7 +107,7 @@ exports.cancelDownload = async (download_uid) => { logger.info(`Download ${download_uid} could not be cancelled before completing.`); return false; } - return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true}); + return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true, running: false}); } exports.clearDownload = async (download_uid) => { @@ -127,7 +128,7 @@ async function fixDownloadState() { const running_downloads = downloads.filter(download => !download['finished'] && !download['error']); for (let i = 0; i < running_downloads.length; i++) { const running_download = running_downloads[i]; - const update_obj = {finished_step: true, paused: true}; + const update_obj = {finished_step: true, paused: true, running: false}; if (running_download['step_index'] > 0) { update_obj['step_index'] = running_download['step_index'] - 1; } @@ -151,18 +152,19 @@ async function checkDownloads() { return; }); + let running_downloads_count = downloads.filter(download => download['running']).length; const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']); for (let i = 0; i < waiting_downloads.length; i++) { - const running_download = waiting_downloads[i]; - if (i === 5/*config_api.getConfigItem('ytdl_max_concurrent_downloads')*/) break; + const waiting_download = waiting_downloads[i]; + if (running_downloads_count >= 5/*config_api.getConfigItem('ytdl_max_concurrent_downloads')*/) break; - if (running_download['finished_step'] && !running_download['finished']) { + if (waiting_download['finished_step'] && !waiting_download['finished']) { // move to next step - - if (running_download['step_index'] === 0) { - collectInfo(running_download['uid']); - } else if (running_download['step_index'] === 1) { - downloadQueuedFile(running_download['uid']); + running_downloads_count++; + if (waiting_download['step_index'] === 0) { + collectInfo(waiting_download['uid']); + } else if (waiting_download['step_index'] === 1) { + downloadQueuedFile(waiting_download['uid']); } } } @@ -174,7 +176,7 @@ async function collectInfo(download_uid) { return; } logger.verbose(`Collecting info for download ${download_uid}`); - await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false, running: true}); const url = download['url']; const type = download['type']; @@ -194,7 +196,7 @@ async function collectInfo(download_uid) { if (!info) { // info failed, record error and pause download const error = 'Failed to get info, see server logs for specific error.'; - await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); return; } @@ -227,6 +229,7 @@ async function collectInfo(download_uid) { const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null; await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args, finished_step: true, + running: false, options: options, files_to_check_for_progress: files_to_check_for_progress, expected_file_size: expected_file_size, @@ -243,7 +246,7 @@ async function downloadQueuedFile(download_uid) { return new Promise(async resolve => { const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); - await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false, running: true}); const url = download['url']; const type = download['type']; @@ -355,7 +358,7 @@ async function downloadQueuedFile(download_uid) { } const file_uids = file_objs.map(file_obj => file_obj.uid); - await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container}); resolve(); } }); @@ -544,7 +547,7 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) { logger.error(`Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`); if (download_uid) { const error = 'Failed to get info, see server logs for specific error.'; - await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); } resolve(null); } @@ -555,7 +558,7 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) { } if (download_uid) { const error = 'Failed to get info, see server logs for specific error.'; - await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true}); + await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); } resolve(null); } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index b2d2f79..385c5fd 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -414,15 +414,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push('--write-thumbnail'); } - const download_delay = config_api.getConfigItem('ytdl_subscriptions_download_delay'); - if (download_delay && downloadConfig.indexOf('--sleep-interval') === -1) { - if (!(+download_delay)) { - logger.warn(`Invalid download delay of ${download_delay}, please remember to use non-zero numbers.`); - } else { - downloadConfig.push('--sleep-interval', +download_delay); - } - } - const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit'); if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) { downloadConfig.push('-r', rate_limit); @@ -440,7 +431,7 @@ async function getFilesToDownload(sub, output_jsons) { const files_to_download = []; for (let i = 0; i < output_jsons.length; i++) { const output_json = output_jsons[i]; - const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})); + const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']})) && !(await db_api.getRecord('download_queue', {sub_id: sub.id, url: output_json['webpage_url'], error: null})); if (file_missing) { const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']}); if (file_with_path_exists) { diff --git a/src/app/components/downloads/downloads.component.html b/src/app/components/downloads/downloads.component.html index 9ba56ec..2c336b7 100644 --- a/src/app/components/downloads/downloads.component.html +++ b/src/app/components/downloads/downloads.component.html @@ -1,16 +1,16 @@ -
+
- + - Date + Date {{element.timestamp_start | date: 'short'}} - Title + Title {{element.title}} @@ -20,7 +20,7 @@ - Subscription + Subscription {{element.sub_name}} @@ -33,13 +33,13 @@ - Stage + Stage {{STEP_INDEX_TO_LABEL[element.step_index]}} - Progress + Progress {{+(element.percent_complete) > 100 ? '100' : element.percent_complete}}% @@ -78,6 +78,9 @@ aria-label="Select page of downloads">
+
+ +
diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index 99bb028..e3e36c1 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -4,6 +4,9 @@ import { trigger, transition, animateChild, stagger, query, style, animate } fro import { Router } from '@angular/router'; import { MatPaginator } from '@angular/material/paginator'; import { MatTableDataSource } from '@angular/material/table'; +import { MatDialog } from '@angular/material/dialog'; +import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component'; +import { MatSort } from '@angular/material/sort'; @Component({ selector: 'app-downloads', @@ -57,13 +60,14 @@ export class DownloadsComponent implements OnInit, OnDestroy { downloads_retrieved = false; @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; sort_downloads = (a, b) => { const result = b.timestamp_start - a.timestamp_start; return result; } - constructor(public postsService: PostsService, private router: Router) { } + constructor(public postsService: PostsService, private router: Router, private dialog: MatDialog) { } ngOnInit(): void { if (this.postsService.initialized) { @@ -103,6 +107,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { this.downloads.sort(this.sort_downloads); this.dataSource = new MatTableDataSource(this.downloads); this.dataSource.paginator = this.paginator; + this.dataSource.sort = this.sort; } else { // failed to get downloads } @@ -110,9 +115,21 @@ export class DownloadsComponent implements OnInit, OnDestroy { } clearFinishedDownloads(): void { - this.postsService.clearFinishedDownloads().subscribe(res => { - if (!res['success']) { - this.postsService.openSnackBar('Failed to clear finished downloads!'); + const dialogRef = this.dialog.open(ConfirmDialogComponent, { + data: { + dialogTitle: $localize`Clear finished downloads`, + dialogText: $localize`Would you like to clear your finished downloads?`, + submitText: $localize`Clear`, + warnSubmitColor: true + } + }); + dialogRef.afterClosed().subscribe(confirmed => { + if (confirmed) { + this.postsService.clearFinishedDownloads().subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to clear finished downloads!'); + } + }); } }); } diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 43d0579..710b857 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -58,12 +58,6 @@ Unit is seconds, only include numbers.
-
- - - Units is seconds, will force youtube-dl to sleep between videos in a subscription by the specified number of seconds. Only include numbers. - -
Redownload fresh uploads