diff --git a/Public API v1.yaml b/Public API v1.yaml index 5be8869..0ca76da 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -881,10 +881,30 @@ paths: application/json: schema: $ref: '#/components/schemas/GetAllTasksResponse' + /api/resetTasks: + post: + tags: + - tasks + summary: Resets all tasks + operationId: post-api-reset-tasks + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/runTask: post: + tags: + - tasks summary: Runs one task operationId: post-api-run-task responses: @@ -901,6 +921,8 @@ paths: $ref: '#/components/schemas/GetTaskRequest' /api/confirmTask: post: + tags: + - tasks summary: Confirms a task operationId: post-api-confirm-task responses: @@ -917,6 +939,8 @@ paths: $ref: '#/components/schemas/GetTaskRequest' /api/cancelTask: post: + tags: + - tasks summary: Cancels a task operationId: post-api-cancel-task responses: @@ -933,6 +957,8 @@ paths: $ref: '#/components/schemas/GetTaskRequest' /api/updateTaskSchedule: post: + tags: + - tasks summary: Updates task schedule operationId: post-api-update-task-schedule responses: @@ -949,6 +975,8 @@ paths: $ref: '#/components/schemas/UpdateTaskScheduleRequest' /api/updateTaskData: post: + tags: + - tasks summary: Updates task data operationId: post-api-update-task-data responses: @@ -965,6 +993,8 @@ paths: $ref: '#/components/schemas/UpdateTaskDataRequest' /api/getDBBackups: post: + tags: + - tasks summary: Get database backups operationId: post-api-get-database-backups responses: @@ -981,6 +1011,8 @@ paths: type: object /api/restoreDBBackup: post: + tags: + - tasks summary: Restore database backup operationId: post-api-restore-database-backup responses: diff --git a/backend/app.js b/backend/app.js index f596b8e..ae71c15 100644 --- a/backend/app.js +++ b/backend/app.js @@ -13,7 +13,6 @@ const unzipper = require('unzipper'); const db_api = require('./db'); const utils = require('./utils') const low = require('lowdb') -const ProgressBar = require('progress'); const fetch = require('node-fetch'); const URL = require('url').URL; const CONSTS = require('./consts') @@ -32,8 +31,7 @@ const tasks_api = require('./tasks'); const subscriptions_api = require('./subscriptions'); const categories_api = require('./categories'); const twitch_api = require('./twitch'); - -const is_windows = process.platform === 'win32'; +const youtubedl_api = require('./youtube-dl'); var app = express(); @@ -357,34 +355,6 @@ async function downloadReleaseFiles(tag) { }); } -// helper function to download file using fetch -async function fetchFile(url, path, file_label) { - var len = null; - const res = await fetch(url); - - len = parseInt(res.headers.get("Content-Length"), 10); - - var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, { - complete: '=', - incomplete: ' ', - width: 20, - total: len - }); - const fileStream = fs.createWriteStream(path); - await new Promise((resolve, reject) => { - res.body.pipe(fileStream); - res.body.on("error", (err) => { - reject(err); - }); - res.body.on('data', function (chunk) { - bar.tick(chunk.length); - }); - fileStream.on("finish", function() { - resolve(); - }); - }); - } - async function downloadReleaseZip(tag) { return new Promise(async resolve => { // get name of zip file, which depends on the version @@ -395,7 +365,7 @@ async function downloadReleaseZip(tag) { let output_path = path.join(__dirname, `youtubedl-material-release-${tag}.zip`); // download zip from release - await fetchFile(latest_zip_link, output_path, 'update ' + tag); + await utils.fetchFile(latest_zip_link, output_path, 'update ' + tag); resolve(true); }); @@ -708,156 +678,8 @@ async function getUrlInfos(url) { async function startYoutubeDL() { // auto update youtube-dl - await autoUpdateYoutubeDL(); -} - -// auto updates the underlying youtube-dl binary, not YoutubeDL-Material -async function autoUpdateYoutubeDL() { - const download_sources = { - 'youtube-dl': { - 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', - 'func': downloadLatestYoutubeDLBinary - }, - 'youtube-dlc': { - 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', - 'func': downloadLatestYoutubeDLCBinary - }, - 'yt-dlp': { - 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', - 'func': downloadLatestYoutubeDLPBinary - } - } - return new Promise(async resolve => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); - const tags_url = download_sources[default_downloader]['tags_url']; - // get current version - let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); - if (!current_app_details_exists) { - logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader}); - } - let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); - let current_version = current_app_details['version']; - let current_downloader = current_app_details['downloader']; - let stored_binary_path = current_app_details['path']; - if (!stored_binary_path || typeof stored_binary_path !== 'string') { - // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); - const guessed_base_path = 'node_modules/youtube-dl/bin/'; - const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); - if (fs.existsSync(guessed_file_path)) { - stored_binary_path = guessed_file_path; - // logger.info('INFO: Guess successful! Update process continuing...') - } else { - logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); - resolve(false); - return; - } - } - - // got version, now let's check the latest version from the youtube-dl API - - - fetch(tags_url, {method: 'Get'}) - .then(async res => res.json()) - .then(async (json) => { - // check if the versions are different - if (!json || !json[0]) { - logger.error(`Failed to check ${default_downloader} version for an update.`) - resolve(false); - return false; - } - const latest_update_version = json[0]['name']; - if (current_version !== latest_update_version || default_downloader !== current_downloader) { - // versions different or different downloader is being used, download new update - logger.info(`Found new update for ${default_downloader}. Updating binary...`); - try { - await checkExistsWithTimeout(stored_binary_path, 10000); - } catch(e) { - logger.error(`Failed to update ${default_downloader} - ${e}`); - } - - await download_sources[default_downloader]['func'](latest_update_version); - - resolve(true); - } else { - resolve(false); - } - }) - .catch(err => { - logger.error(`Failed to check ${default_downloader} version for an update.`) - logger.error(err); - resolve(false); - return false; - }); - }); -} - -async function downloadLatestYoutubeDLBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await fetchFile(download_url, output_path, `youtube-dl ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dl'); -} - -async function downloadLatestYoutubeDLCBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await fetchFile(download_url, output_path, `youtube-dlc ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dlc'); -} - -async function downloadLatestYoutubeDLPBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await fetchFile(download_url, output_path, `yt-dlp ${new_version}`); - - updateDetailsJSON(new_version, 'yt-dlp'); -} - -function updateDetailsJSON(new_version, downloader) { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (new_version) details_json['version'] = new_version; - details_json['downloader'] = downloader; - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); -} - -async function checkExistsWithTimeout(filePath, timeout) { - return new Promise(function (resolve, reject) { - - var timer = setTimeout(function () { - if (watcher) watcher.close(); - reject(new Error('File did not exists and was not created during the timeout.')); - }, timeout); - - fs.access(filePath, fs.constants.R_OK, function (err) { - if (!err) { - clearTimeout(timer); - if (watcher) watcher.close(); - resolve(); - } - }); - - var dir = path.dirname(filePath); - var basename = path.basename(filePath); - var watcher = fs.watch(dir, function (eventType, filename) { - if (eventType === 'rename' && filename === basename) { - clearTimeout(timer); - if (watcher) watcher.close(); - resolve(); - } - }); - }); + const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); + if (update_available) await youtubedl_api.updateYoutubeDL(update_available); } app.use(function(req, res, next) { @@ -1886,6 +1708,17 @@ app.post('/api/getTasks', optionalJwt, async (req, res) => { res.send({tasks: tasks}); }); +app.post('/api/resetTasks', optionalJwt, async (req, res) => { + const tasks_keys = Object.keys(tasks_api.TASKS); + for (let i = 0; i < tasks_keys.length; i++) { + const task_key = tasks_keys[i]; + tasks_api.TASKS[task_key]['job'] = null; + } + await db_api.removeAllRecords('tasks'); + await tasks_api.setupTasks(); + res.send({success: true}); +}); + app.post('/api/getTask', optionalJwt, async (req, res) => { const task_key = req.body.task_key; const task = await db_api.getRecord('tasks', {key: task_key}); diff --git a/backend/tasks.js b/backend/tasks.js index 423f85a..2014da3 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -1,5 +1,5 @@ -const utils = require('./utils'); const db_api = require('./db'); +const youtubedl_api = require('./youtube-dl'); const fs = require('fs-extra'); const logger = require('./logger'); @@ -27,6 +27,12 @@ const TASKS = { confirm: removeDuplicates, title: 'Find duplicate files in DB', job: null + }, + youtubedl_update_check: { + run: youtubedl_api.checkForYoutubeDLUpdate, + confirm: youtubedl_api.updateYoutubeDL, + title: 'Update youtube-dl', + job: null } } @@ -52,20 +58,22 @@ function scheduleJob(task_key, schedule) { return; } + // remove schedule if it's a one-time task + if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null}); // we're just "running" the task, any confirmation should be user-initiated exports.executeRun(task_key); }); } if (db_api.database_initialized) { - setupTasks(); + exports.setupTasks(); } else { db_api.database_initialized_bs.subscribe(init => { - if (init) setupTasks(); + if (init) exports.setupTasks(); }); } -const setupTasks = async () => { +exports.setupTasks = async () => { const tasks_keys = Object.keys(TASKS); for (let i = 0; i < tasks_keys.length; i++) { const task_key = tasks_keys[i]; @@ -90,6 +98,11 @@ const setupTasks = async () => { // schedule task and save job if (task_in_db['schedule']) { + // prevent timestamp schedules from being set to the past + if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) { + await db_api.updateRecord('tasks', {key: task_key}, {schedule: null}); + continue; + } TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']); } } @@ -113,7 +126,7 @@ exports.executeRun = async (task_key) => { // don't set running to true when backup up DB as it will be stick "running" if restored if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true}); const data = await TASKS[task_key].run(); - await db_api.updateRecord('tasks', {key: task_key}, {data: data, last_ran: Date.now()/1000, running: false}); + await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false}); logger.verbose(`Finished running task ${task_key}`); } diff --git a/backend/utils.js b/backend/utils.js index 4f94388..0271ed0 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -1,10 +1,13 @@ -const fs = require('fs-extra') -const path = require('path') +const fs = require('fs-extra'); +const path = require('path'); const ffmpeg = require('fluent-ffmpeg'); +const archiver = require('archiver'); +const fetch = require('node-fetch'); +const ProgressBar = require('progress'); + const config_api = require('./config'); const logger = require('./logger'); -const CONSTS = require('./consts') -const archiver = require('archiver'); +const CONSTS = require('./consts'); const is_windows = process.platform === 'win32'; @@ -356,6 +359,62 @@ async function cropFile(file_path, start, end, ext) { }); } +async function checkExistsWithTimeout(filePath, timeout) { + return new Promise(function (resolve, reject) { + + var timer = setTimeout(function () { + if (watcher) watcher.close(); + reject(new Error('File did not exists and was not created during the timeout.')); + }, timeout); + + fs.access(filePath, fs.constants.R_OK, function (err) { + if (!err) { + clearTimeout(timer); + if (watcher) watcher.close(); + resolve(); + } + }); + + var dir = path.dirname(filePath); + var basename = path.basename(filePath); + var watcher = fs.watch(dir, function (eventType, filename) { + if (eventType === 'rename' && filename === basename) { + clearTimeout(timer); + if (watcher) watcher.close(); + resolve(); + } + }); + }); +} + +// helper function to download file using fetch +async function fetchFile(url, path, file_label) { + var len = null; + const res = await fetch(url); + + len = parseInt(res.headers.get("Content-Length"), 10); + + var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, { + complete: '=', + incomplete: ' ', + width: 20, + total: len + }); + const fileStream = fs.createWriteStream(path); + await new Promise((resolve, reject) => { + res.body.pipe(fileStream); + res.body.on("error", (err) => { + reject(err); + }); + res.body.on('data', function (chunk) { + bar.tick(chunk.length); + }); + fileStream.on("finish", function() { + resolve(); + }); + }); +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { @@ -397,5 +456,7 @@ module.exports = { cropFile: cropFile, createEdgeNGrams: createEdgeNGrams, wait: wait, + checkExistsWithTimeout: checkExistsWithTimeout, + fetchFile: fetchFile, File: File } diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js new file mode 100644 index 0000000..80432fb --- /dev/null +++ b/backend/youtube-dl.js @@ -0,0 +1,127 @@ +const fs = require('fs-extra'); +const fetch = require('node-fetch'); + +const logger = require('./logger'); +const utils = require('./utils'); +const CONSTS = require('./consts'); +const config_api = require('./config.js'); + +const is_windows = process.platform === 'win32'; + +const download_sources = { + 'youtube-dl': { + 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', + 'func': downloadLatestYoutubeDLBinary + }, + 'youtube-dlc': { + 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', + 'func': downloadLatestYoutubeDLCBinary + }, + 'yt-dlp': { + 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', + 'func': downloadLatestYoutubeDLPBinary + } +} + +exports.checkForYoutubeDLUpdate = async () => { + return new Promise(async resolve => { + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + const tags_url = download_sources[default_downloader]['tags_url']; + // get current version + let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); + if (!current_app_details_exists) { + logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader}); + } + let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); + let current_version = current_app_details['version']; + let current_downloader = current_app_details['downloader']; + let stored_binary_path = current_app_details['path']; + if (!stored_binary_path || typeof stored_binary_path !== 'string') { + // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); + const guessed_base_path = 'node_modules/youtube-dl/bin/'; + const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); + if (fs.existsSync(guessed_file_path)) { + stored_binary_path = guessed_file_path; + // logger.info('INFO: Guess successful! Update process continuing...') + } else { + logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); + resolve(null); + return; + } + } + + // got version, now let's check the latest version from the youtube-dl API + + + fetch(tags_url, {method: 'Get'}) + .then(async res => res.json()) + .then(async (json) => { + // check if the versions are different + if (!json || !json[0]) { + logger.error(`Failed to check ${default_downloader} version for an update.`) + resolve(null); + return; + } + const latest_update_version = json[0]['name']; + if (current_version !== latest_update_version || default_downloader !== current_downloader) { + // versions different or different downloader is being used, download new update + resolve(latest_update_version); + } else { + resolve(null); + } + return; + }) + .catch(err => { + logger.error(`Failed to check ${default_downloader} version for an update.`) + logger.error(err); + resolve(null); + return; + }); + }); +} + +exports.updateYoutubeDL = async (latest_update_version) => { + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + await download_sources[default_downloader]['func'](latest_update_version); +} + +async function downloadLatestYoutubeDLBinary(new_version) { + const file_ext = is_windows ? '.exe' : ''; + + const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`; + const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`); + + updateDetailsJSON(new_version, 'youtube-dl'); +} + +async function downloadLatestYoutubeDLCBinary(new_version) { + const file_ext = is_windows ? '.exe' : ''; + + const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`; + const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`); + + updateDetailsJSON(new_version, 'youtube-dlc'); +} + +async function downloadLatestYoutubeDLPBinary(new_version) { + const file_ext = is_windows ? '.exe' : ''; + + const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`; + const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`); + + updateDetailsJSON(new_version, 'yt-dlp'); +} + +function updateDetailsJSON(new_version, downloader) { + const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); + if (new_version) details_json['version'] = new_version; + details_json['downloader'] = downloader; + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); +} diff --git a/src/app/components/tasks/tasks.component.html b/src/app/components/tasks/tasks.component.html index f883f54..662e3be 100644 --- a/src/app/components/tasks/tasks.component.html +++ b/src/app/components/tasks/tasks.component.html @@ -50,11 +50,18 @@
-
+
@@ -80,6 +87,7 @@
+
diff --git a/src/app/components/tasks/tasks.component.ts b/src/app/components/tasks/tasks.component.ts index cc86909..8fec372 100644 --- a/src/app/components/tasks/tasks.component.ts +++ b/src/app/components/tasks/tasks.component.ts @@ -3,6 +3,7 @@ import { MatDialog } from '@angular/material/dialog'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTableDataSource } from '@angular/material/table'; +import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component'; import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component'; import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component'; import { PostsService } from 'app/posts.services'; @@ -125,6 +126,31 @@ export class TasksComponent implements OnInit { }) } + resetTasks(): void { + const dialogRef = this.dialog.open(ConfirmDialogComponent, { + data: { + dialogTitle: $localize`Reset tasks`, + dialogText: $localize`Would you like to reset your tasks? All your schedules will be removed as well.`, + submitText: $localize`Reset`, + warnSubmitColor: true + } + }); + dialogRef.afterClosed().subscribe(confirmed => { + if (confirmed) { + this.postsService.resetTasks().subscribe(res => { + if (res['success']) { + this.postsService.openSnackBar($localize`Tasks successfully reset!`); + } else { + this.postsService.openSnackBar($localize`Failed to reset tasks!`); + } + }, err => { + this.postsService.openSnackBar($localize`Failed to reset tasks!`); + console.error(err); + }); + } + }); + } + } export interface Task { diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 68f9890..6df00e5 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -591,6 +591,10 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getTasks', {}, this.httpOptions); } + resetTasks() { + return this.http.post(this.path + 'resetTasks', {}, this.httpOptions); + } + getTask(task_key: string) { const body: GetTaskRequest = {task_key: task_key}; return this.http.post(this.path + 'getTask', body, this.httpOptions);