From c207e568557640ea6fdb6139d99070e1af8e9460 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Tue, 23 May 2023 22:37:25 -0400 Subject: [PATCH] Rebuild database task (#900) * Improved tests for multi-user mode * Adds task to rebuild database * Updated subscriptions.js export syntax * Subscription metadata is now backed up * Added typing to task key * Updated api models * Tasks actions styling update --- Public API v1.yaml | 21 +++- backend/app.js | 45 ++++++--- backend/authentication/auth.js | 96 ++++++++----------- backend/consts.js | 2 + backend/subscriptions.js | 59 ++++++------ backend/tasks.js | 78 ++++++++++++++- backend/test/tests.js | 26 +++-- backend/utils.js | 11 +++ src/api-types/index.ts | 1 + src/api-types/models/Config.ts | 2 +- src/api-types/models/Download.ts | 2 +- .../models/GetFileFormatsResponse.ts | 2 +- .../models/GetSubscriptionResponse.ts | 2 +- src/api-types/models/GetTaskRequest.ts | 4 +- src/api-types/models/Notification.ts | 2 +- src/api-types/models/Subscription.ts | 2 +- src/api-types/models/Task.ts | 7 +- src/api-types/models/TaskType.ts | 14 +++ src/api-types/models/UpdateFileRequest.ts | 2 +- src/api-types/models/UpdateTaskDataRequest.ts | 6 +- .../models/UpdateTaskOptionsRequest.ts | 6 +- .../models/UpdateTaskScheduleRequest.ts | 3 +- .../task-settings/task-settings.component.ts | 4 +- src/app/components/tasks/tasks.component.html | 2 +- src/app/components/tasks/tasks.component.ts | 37 ++++++- src/app/posts.services.ts | 15 +-- 26 files changed, 305 insertions(+), 146 deletions(-) create mode 100644 src/api-types/models/TaskType.ts diff --git a/Public API v1.yaml b/Public API v1.yaml index 4ddf86e..e49783d 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -1758,14 +1758,14 @@ components: type: object properties: task_key: - type: string + $ref: '#/components/schemas/TaskType' required: - task_key UpdateTaskScheduleRequest: type: object properties: task_key: - type: string + $ref: '#/components/schemas/TaskType' new_schedule: $ref: '#/components/schemas/Schedule' required: @@ -1775,7 +1775,7 @@ components: type: object properties: task_key: - type: string + $ref: '#/components/schemas/TaskType' new_data: type: object required: @@ -1785,7 +1785,7 @@ components: type: object properties: task_key: - type: string + $ref: '#/components/schemas/TaskType' new_options: type: object required: @@ -2726,7 +2726,7 @@ components: type: object properties: key: - type: string + $ref: '#/components/schemas/TaskType' title: type: string last_ran: @@ -2745,6 +2745,17 @@ components: $ref: '#/components/schemas/Schedule' options: type: object + TaskType: + type: string + enum: + - backup_local_db + - missing_files_check + - missing_db_records + - duplicate_files_check + - youtubedl_update_check + - delete_old_files + - import_legacy_archives + - rebuild_database Schedule: required: - type diff --git a/backend/app.js b/backend/app.js index 28f3fca..049d9bc 100644 --- a/backend/app.js +++ b/backend/app.js @@ -535,6 +535,7 @@ async function loadConfig() { if (allowSubscriptions) { // set downloading to false let subscriptions = await subscriptions_api.getAllSubscriptions(); + subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub)); subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false}); // runs initially, then runs every ${subscriptionCheckInterval} seconds const watchSubscriptionsInterval = function() { @@ -1928,9 +1929,34 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) { // user authentication -app.post('/api/auth/register' - , optionalJwt - , auth_api.registerUser); +app.post('/api/auth/register', optionalJwt, async (req, res) => { + const userid = req.body.userid; + const username = req.body.username; + const plaintextPassword = req.body.password; + + if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) { + logger.error(`Registration failed for user ${userid}. Registration is disabled.`); + res.sendStatus(409); + return; + } + + if (plaintextPassword === "") { + logger.error(`Registration failed for user ${userid}. A password must be provided.`); + res.sendStatus(409); + return; + } + + const new_user = await auth_api.registerUser(userid, username, plaintextPassword); + + if (!new_user) { + res.sendStatus(409); + return; + } + + res.send({ + user: new_user + }); +}); app.post('/api/auth/login' , auth_api.passport.authenticate(['local', 'ldapauth'], {}) , auth_api.generateJWT @@ -1982,18 +2008,7 @@ app.post('/api/updateUser', optionalJwt, async (req, res) => { app.post('/api/deleteUser', optionalJwt, async (req, res) => { let uid = req.body.uid; try { - let success = false; - let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); - const user_folder = path.join(__dirname, usersFileFolder, uid); - const user_db_obj = await db_api.getRecord('users', {uid: uid}); - if (user_db_obj) { - // user exists, let's delete - await fs.remove(user_folder); - await db_api.removeRecord('users', {uid: uid}); - success = true; - } else { - logger.error(`Could not find user with uid ${uid}`); - } + const success = await auth_api.deleteUser(uid); res.send({success: success}); } catch (err) { logger.error(err); diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index c630c1b..360e7c3 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -1,11 +1,13 @@ const config_api = require('../config'); -const consts = require('../consts'); +const CONSTS = require('../consts'); const logger = require('../logger'); const db_api = require('../db'); const jwt = require('jsonwebtoken'); const { uuid } = require('uuidv4'); const bcrypt = require('bcryptjs'); +const fs = require('fs-extra'); +const path = require('path'); var LocalStrategy = require('passport-local').Strategy; var LdapStrategy = require('passport-ldapauth'); @@ -16,7 +18,7 @@ var JwtStrategy = require('passport-jwt').Strategy, let SERVER_SECRET = null; let JWT_EXPIRATION = null; let opts = null; -let saltRounds = null; +let saltRounds = 10; exports.initialize = function () { /************************* @@ -31,8 +33,6 @@ exports.initialize = function () { }); } - saltRounds = 10; - // Sometimes this value is not properly typed: https://github.com/Tzahi12345/YoutubeDL-Material/issues/813 JWT_EXPIRATION = config_api.getConfigItem('ytdl_jwt_expiration'); if (!(+JWT_EXPIRATION)) { @@ -68,7 +68,7 @@ exports.initialize = function () { const setupRoles = async () => { const required_roles = { admin: { - permissions: consts.AVAILABLE_PERMISSIONS + permissions: CONSTS.AVAILABLE_PERMISSIONS }, user: { permissions: [ @@ -106,55 +106,41 @@ exports.passport.deserializeUser(function(user, done) { /*************************************** * Register user with hashed password **************************************/ -exports.registerUser = async function(req, res) { - var userid = req.body.userid; - var username = req.body.username; - var plaintextPassword = req.body.password; - - if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) { - res.sendStatus(409); - logger.error(`Registration failed for user ${userid}. Registration is disabled.`); - return; - } - if (plaintextPassword === "") { - res.sendStatus(400); - logger.error(`Registration failed for user ${userid}. A password must be provided.`); - return; +exports.registerUser = async (userid, username, plaintextPassword) => { + const hash = await bcrypt.hash(plaintextPassword, saltRounds); + const new_user = generateUserObject(userid, username, hash); + // check if user exists + if (await db_api.getRecord('users', {uid: userid})) { + // user id is taken! + logger.error('Registration failed: UID is already taken!'); + return null; + } else if (await db_api.getRecord('users', {name: username})) { + // user name is taken! + logger.error('Registration failed: User name is already taken!'); + return null; + } else { + // add to db + await db_api.insertRecordIntoTable('users', new_user); + logger.verbose(`New user created: ${new_user.name}`); + return new_user; } +} - bcrypt.hash(plaintextPassword, saltRounds) - .then(async function(hash) { - let new_user = generateUserObject(userid, username, hash); - // check if user exists - if (await db_api.getRecord('users', {uid: userid})) { - // user id is taken! - logger.error('Registration failed: UID is already taken!'); - res.status(409).send('UID is already taken!'); - } else if (await db_api.getRecord('users', {name: username})) { - // user name is taken! - logger.error('Registration failed: User name is already taken!'); - res.status(409).send('User name is already taken!'); - } else { - // add to db - await db_api.insertRecordIntoTable('users', new_user); - logger.verbose(`New user created: ${new_user.name}`); - res.send({ - user: new_user - }); - } - }) - .then(function(result) { - - }) - .catch(function(err) { - logger.error(err); - if( err.code == 'ER_DUP_ENTRY' ) { - res.status(409).send('UserId already taken'); - } else { - res.sendStatus(409); - } - }); +exports.deleteUser = async (uid) => { + let success = false; + let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + const user_folder = path.join(__dirname, usersFileFolder, uid); + const user_db_obj = await db_api.getRecord('users', {uid: uid}); + if (user_db_obj) { + // user exists, let's delete + await fs.remove(user_folder); + await db_api.removeRecord('users', {uid: uid}); + success = true; + } else { + logger.error(`Could not find user with uid ${uid}`); + } + return success; } /*************************************** @@ -235,7 +221,7 @@ exports.returnAuthResponse = async function(req, res) { user: req.user, token: req.token, permissions: await exports.userPermissions(req.user.uid), - available_permissions: consts['AVAILABLE_PERMISSIONS'] + available_permissions: CONSTS.AVAILABLE_PERMISSIONS }); } @@ -319,7 +305,7 @@ exports.getUserVideos = async function(user_uid, type) { } exports.getUserVideo = async function(user_uid, file_uid, requireSharing = false) { - let file = await db_api.getRecord('files', {file_uid: file_uid}); + let file = await db_api.getRecord('files', {uid: file_uid}); // prevent unauthorized users from accessing the file info if (file && !file['sharingEnabled'] && requireSharing) file = null; @@ -406,8 +392,8 @@ exports.userPermissions = async function(user_uid) { const role_obj = await db_api.getRecord('roles', {key: role}); const role_permissions = role_obj['permissions']; - for (let i = 0; i < consts['AVAILABLE_PERMISSIONS'].length; i++) { - let permission = consts['AVAILABLE_PERMISSIONS'][i]; + for (let i = 0; i < CONSTS.AVAILABLE_PERMISSIONS.length; i++) { + let permission = CONSTS.AVAILABLE_PERMISSIONS[i]; const user_has_explicit_permission = user_obj['permissions'].includes(permission); const permission_in_overrides = user_obj['permission_overrides'].includes(permission); diff --git a/backend/consts.js b/backend/consts.js index 4461819..6a36d10 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -347,6 +347,8 @@ const YTDL_ARGS_WITH_VALUES = [ '--convert-subs' ]; +exports.SUBSCRIPTION_BACKUP_PATH = 'subscription_backup.json' + // we're using a Set here for performance exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES); diff --git a/backend/subscriptions.js b/backend/subscriptions.js index e5ffa84..03c07b4 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -6,20 +6,21 @@ const config_api = require('./config'); const archive_api = require('./archive'); const utils = require('./utils'); const logger = require('./logger'); +const CONSTS = require('./consts'); const debugMode = process.env.YTDL_MODE === 'debug'; const db_api = require('./db'); const downloader_api = require('./downloader'); -async function subscribe(sub, user_uid = null) { +exports.subscribe = async (sub, user_uid = null, skip_get_info = false) => { const result_obj = { success: false, error: '' }; return new Promise(async resolve => { // sub should just have url and name. here we will get isPlaylist and path - sub.isPlaylist = sub.url.includes('playlist'); + sub.isPlaylist = sub.isPlaylist || sub.url.includes('playlist'); sub.videos = []; let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid})); @@ -34,10 +35,11 @@ async function subscribe(sub, user_uid = null) { sub['user_uid'] = user_uid ? user_uid : undefined; await db_api.insertRecordIntoTable('subscriptions', sub); - let success = await getSubscriptionInfo(sub); + let success = skip_get_info ? true : await getSubscriptionInfo(sub); + exports.writeSubscriptionMetadata(sub); if (success) { - getVideosForSub(sub, user_uid); + if (!sub.paused) exports.getVideosForSub(sub, user_uid); } else { logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') } @@ -109,7 +111,7 @@ async function getSubscriptionInfo(sub) { }); } -async function unsubscribe(sub, deleteMode, user_uid = null) { +exports.unsubscribe = async (sub, deleteMode, user_uid = null) => { let basePath = null; if (user_uid) basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); @@ -148,7 +150,7 @@ async function unsubscribe(sub, deleteMode, user_uid = null) { await db_api.removeAllRecords('archives', {sub_id: sub.id}); } -async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) { +exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = null, user_uid = null) => { if (typeof sub === 'string') { // TODO: fix bad workaround where sub is a sub_id sub = await db_api.getRecord('subscriptions', {sub_id: sub}); @@ -216,8 +218,8 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, } } -async function getVideosForSub(sub, user_uid = null) { - const latest_sub_obj = await getSubscription(sub.id); +exports.getVideosForSub = async (sub, user_uid = null) => { + const latest_sub_obj = await exports.getSubscription(sub.id); if (!latest_sub_obj || latest_sub_obj['downloading']) { return false; } @@ -305,7 +307,7 @@ async function handleOutputJSON(output, sub, user_uid) { } const files_to_download = await getFilesToDownload(sub, output_jsons); - const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid); + const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid); for (let j = 0; j < files_to_download.length; j++) { const file_to_download = files_to_download[j]; @@ -316,7 +318,7 @@ async function handleOutputJSON(output, sub, user_uid) { return files_to_download; } -function generateOptionsForSubscriptionDownload(sub, user_uid) { +exports.generateOptionsForSubscriptionDownload = (sub, user_uid) => { let basePath = null; if (user_uid) basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); @@ -443,35 +445,36 @@ async function getFilesToDownload(sub, output_jsons) { } -async function getSubscriptions(user_uid = null) { +exports.getSubscriptions = async (user_uid = null) => { return await db_api.getRecords('subscriptions', {user_uid: user_uid}); } -async function getAllSubscriptions() { +exports.getAllSubscriptions = async () => { const all_subs = await db_api.getRecords('subscriptions'); const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode); } -async function getSubscription(subID) { +exports.getSubscription = async (subID) => { // stringify and parse because we may override the 'downloading' property const sub = JSON.parse(JSON.stringify(await db_api.getRecord('subscriptions', {id: subID}))); // now with the download_queue, we may need to override 'downloading' - const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: sub.id}, true); + const current_downloads = await db_api.getRecords('download_queue', {running: true, sub_id: subID}, true); if (!sub['downloading']) sub['downloading'] = current_downloads > 0; return sub; } -async function getSubscriptionByName(subName, user_uid = null) { +exports.getSubscriptionByName = async (subName, user_uid = null) => { return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid}); } -async function updateSubscription(sub) { +exports.updateSubscription = async (sub) => { await db_api.updateRecord('subscriptions', {id: sub.id}, sub); + exports.writeSubscriptionMetadata(sub); return true; } -async function updateSubscriptionPropertyMultiple(subs, assignment_obj) { +exports.updateSubscriptionPropertyMultiple = async (subs, assignment_obj) => { subs.forEach(async sub => { await updateSubscriptionProperty(sub, assignment_obj); }); @@ -483,6 +486,14 @@ async function updateSubscriptionProperty(sub, assignment_obj) { return true; } +exports.writeSubscriptionMetadata = (sub) => { + let basePath = sub.user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), sub.user_uid, 'subscriptions') + : config_api.getConfigItem('ytdl_subscriptions_base_path'); + const appendedBasePath = getAppendedBasePath(sub, basePath); + const metadata_path = path.join(appendedBasePath, CONSTS.SUBSCRIPTION_BACKUP_PATH); + fs.writeJSONSync(metadata_path, sub); +} + async function setFreshUploads(sub) { const sub_files = await db_api.getRecords('files', {sub_id: sub.id}); if (!sub_files) return; @@ -537,17 +548,3 @@ async function checkVideoIfBetterExists(file_obj, sub, user_uid) { function getAppendedBasePath(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } - -module.exports = { - getSubscription : getSubscription, - getSubscriptionByName : getSubscriptionByName, - getSubscriptions : getSubscriptions, - getAllSubscriptions : getAllSubscriptions, - updateSubscription : updateSubscription, - subscribe : subscribe, - unsubscribe : unsubscribe, - deleteSubscriptionFile : deleteSubscriptionFile, - getVideosForSub : getVideosForSub, - updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple, - generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload -} diff --git a/backend/tasks.js b/backend/tasks.js index e7a3493..b5c899e 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -3,10 +3,17 @@ const notifications_api = require('./notifications'); const youtubedl_api = require('./youtube-dl'); const archive_api = require('./archive'); const files_api = require('./files'); +const subscriptions_api = require('./subscriptions'); +const config_api = require('./config'); +const auth_api = require('./authentication/auth'); +const utils = require('./utils'); +const logger = require('./logger'); +const CONSTS = require('./consts'); const fs = require('fs-extra'); -const logger = require('./logger'); +const path = require('path'); const scheduler = require('node-schedule'); +const { uuid } = require('uuidv4'); const TASKS = { backup_local_db: { @@ -47,6 +54,11 @@ const TASKS = { run: archive_api.importArchives, title: 'Import legacy archives', job: null + }, + rebuild_database: { + run: rebuildDB, + title: 'Rebuild database', + job: null } } @@ -265,4 +277,68 @@ async function autoDeleteFiles(data) { } } +async function rebuildDB() { + await db_api.backupDB(); + let subs_to_add = await guessSubscriptions(false); + subs_to_add = subs_to_add.concat(await guessSubscriptions(true)); + const users_to_add = await guessUsers(); + for (const user_to_add of users_to_add) { + const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + + const user_exists = await db_api.getRecord('users', {uid: user_to_add}); + if (!user_exists) { + await auth_api.registerUser(user_to_add, user_to_add, 'password'); + logger.info(`Regenerated user ${user_to_add}`); + } + + const user_channel_subs = await guessSubscriptions(false, path.join(usersFileFolder, user_to_add), user_to_add); + const user_playlist_subs = await guessSubscriptions(true, path.join(usersFileFolder, user_to_add), user_to_add); + subs_to_add = subs_to_add.concat(user_channel_subs, user_playlist_subs); + } + + for (const sub_to_add of subs_to_add) { + const sub_exists = !!(await subscriptions_api.getSubscriptionByName(sub_to_add['name'], sub_to_add['user_uid'])); + // TODO: we shouldn't be creating this here + const new_sub = Object.assign({}, sub_to_add, {paused: true}); + if (!sub_exists) { + await subscriptions_api.subscribe(new_sub, sub_to_add['user_uid'], true); + logger.info(`Regenerated subscription ${sub_to_add['name']}`); + } + } + + logger.info(`Importing unregistered files`); + await files_api.importUnregisteredFiles(); +} + +const guessUsers = async () => { + const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + const userPaths = await utils.getDirectoriesInDirectory(usersFileFolder); + return userPaths.map(userPath => path.basename(userPath)); +} + +const guessSubscriptions = async (isPlaylist, basePath = null) => { + const guessed_subs = []; + const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + const subsSubPath = basePath ? path.join(basePath, 'subscriptions') : subscriptionsFileFolder; + const subsPath = path.join(subsSubPath, isPlaylist ? 'playlists' : 'channels'); + + const subs = await utils.getDirectoriesInDirectory(subsPath); + for (const subPath of subs) { + const sub_backup_path = path.join(subPath, CONSTS.SUBSCRIPTION_BACKUP_PATH); + if (!fs.existsSync(sub_backup_path)) continue; + + try { + const sub_backup = fs.readJSONSync(sub_backup_path) + delete sub_backup['_id']; + guessed_subs.push(sub_backup); + } catch(err) { + logger.warn(`Failed to reimport subscription in path ${subPath}`) + logger.warn(err); + } + } + + return guessed_subs; +} + exports.TASKS = TASKS; \ No newline at end of file diff --git a/backend/test/tests.js b/backend/test/tests.js index 3b79860..51fe9fd 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -337,16 +337,22 @@ describe('Database', async function() { }); describe('Multi User', async function() { - let user = null; - const user_to_test = 'admin'; - const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; - const playlist_to_test = 'ysabVZz4x'; + const user_to_test = 'test_user'; + const user_password = 'test_pass'; + const sub_to_test = ''; + const playlist_to_test = ''; beforeEach(async function() { await db_api.connectToDB(); - user = await auth_api.login('admin', 'pass'); + await auth_api.deleteUser(user_to_test); }); - describe('Authentication', function() { - it('login', async function() { + describe('Basic', function() { + it('Register', async function() { + const user = await auth_api.registerUser(user_to_test, user_to_test, user_password); + assert(user); + }); + it('Login', async function() { + await auth_api.registerUser(user_to_test, user_to_test, user_password); + const user = await auth_api.login(user_to_test, user_password); assert(user); }); }); @@ -362,14 +368,14 @@ describe('Multi User', async function() { }); it('Video access - disallowed', async function() { - await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}, user_to_test); - const video_obj = auth_api.getUserVideo('admin', video_to_test, true); + await db_api.setVideoProperty(video_to_test, {sharingEnabled: false}); + const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true); assert(!video_obj); }); it('Video access - allowed', async function() { await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test); - const video_obj = auth_api.getUserVideo('admin', video_to_test, true); + const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true); assert(video_obj); }); }); diff --git a/backend/utils.js b/backend/utils.js index eafb946..157b6f0 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -519,6 +519,17 @@ exports.convertFlatObjectToNestedObject = (obj) => { return result; } +exports.getDirectoriesInDirectory = async (basePath) => { + try { + const files = await fs.readdir(basePath, { withFileTypes: true }); + return files + .filter((file) => file.isDirectory()) + .map((file) => path.join(basePath, file.name)); + } catch (err) { + return []; + } +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { diff --git a/src/api-types/index.ts b/src/api-types/index.ts index 5da5b0a..c749b0b 100644 --- a/src/api-types/index.ts +++ b/src/api-types/index.ts @@ -104,6 +104,7 @@ export type { SubscriptionRequestData } from './models/SubscriptionRequestData'; export type { SuccessObject } from './models/SuccessObject'; export type { TableInfo } from './models/TableInfo'; export type { Task } from './models/Task'; +export { TaskType } from './models/TaskType'; export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest'; export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse'; export type { TransferDBRequest } from './models/TransferDBRequest'; diff --git a/src/api-types/models/Config.ts b/src/api-types/models/Config.ts index cb80587..9bf8057 100644 --- a/src/api-types/models/Config.ts +++ b/src/api-types/models/Config.ts @@ -3,5 +3,5 @@ /* eslint-disable */ export type Config = { - YoutubeDLMaterial: Record; + YoutubeDLMaterial: any; }; diff --git a/src/api-types/models/Download.ts b/src/api-types/models/Download.ts index 1a0d05e..84d95e5 100644 --- a/src/api-types/models/Download.ts +++ b/src/api-types/models/Download.ts @@ -26,5 +26,5 @@ export type Download = { user_uid?: string; sub_id?: string; sub_name?: string; - prefetched_info?: Record; + prefetched_info?: any; }; diff --git a/src/api-types/models/GetFileFormatsResponse.ts b/src/api-types/models/GetFileFormatsResponse.ts index 34e5059..0412637 100644 --- a/src/api-types/models/GetFileFormatsResponse.ts +++ b/src/api-types/models/GetFileFormatsResponse.ts @@ -5,6 +5,6 @@ export type GetFileFormatsResponse = { success: boolean; result: { -formats?: Array>; +formats?: Array; }; }; diff --git a/src/api-types/models/GetSubscriptionResponse.ts b/src/api-types/models/GetSubscriptionResponse.ts index 3801205..c9931c6 100644 --- a/src/api-types/models/GetSubscriptionResponse.ts +++ b/src/api-types/models/GetSubscriptionResponse.ts @@ -6,5 +6,5 @@ import type { Subscription } from './Subscription'; export type GetSubscriptionResponse = { subscription: Subscription; - files: Array>; + files: Array; }; diff --git a/src/api-types/models/GetTaskRequest.ts b/src/api-types/models/GetTaskRequest.ts index a188892..e862311 100644 --- a/src/api-types/models/GetTaskRequest.ts +++ b/src/api-types/models/GetTaskRequest.ts @@ -2,6 +2,8 @@ /* tslint:disable */ /* eslint-disable */ +import type { TaskType } from './TaskType'; + export type GetTaskRequest = { - task_key: string; + task_key: TaskType; }; diff --git a/src/api-types/models/Notification.ts b/src/api-types/models/Notification.ts index 6ad7425..f77e917 100644 --- a/src/api-types/models/Notification.ts +++ b/src/api-types/models/Notification.ts @@ -11,6 +11,6 @@ export type Notification = { user_uid?: string; action?: Array; read: boolean; - data?: Record; + data?: any; timestamp: number; }; diff --git a/src/api-types/models/Subscription.ts b/src/api-types/models/Subscription.ts index bffb738..9d84e2c 100644 --- a/src/api-types/models/Subscription.ts +++ b/src/api-types/models/Subscription.ts @@ -15,5 +15,5 @@ export type Subscription = { timerange?: string; custom_args?: string; custom_output?: string; - videos: Array>; + videos: Array; }; diff --git a/src/api-types/models/Task.ts b/src/api-types/models/Task.ts index a9dbacc..ec9befd 100644 --- a/src/api-types/models/Task.ts +++ b/src/api-types/models/Task.ts @@ -3,16 +3,17 @@ /* eslint-disable */ import type { Schedule } from './Schedule'; +import type { TaskType } from './TaskType'; export type Task = { - key: string; + key: TaskType; title?: string; last_ran: number; last_confirmed: number; running: boolean; confirming: boolean; - data: Record; + data: any; error: string; schedule: Schedule; - options?: Record; + options?: any; }; diff --git a/src/api-types/models/TaskType.ts b/src/api-types/models/TaskType.ts new file mode 100644 index 0000000..a3b2e7d --- /dev/null +++ b/src/api-types/models/TaskType.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export enum TaskType { + BACKUP_LOCAL_DB = 'backup_local_db', + MISSING_FILES_CHECK = 'missing_files_check', + MISSING_DB_RECORDS = 'missing_db_records', + DUPLICATE_FILES_CHECK = 'duplicate_files_check', + YOUTUBEDL_UPDATE_CHECK = 'youtubedl_update_check', + DELETE_OLD_FILES = 'delete_old_files', + IMPORT_LEGACY_ARCHIVES = 'import_legacy_archives', + REBUILD_DATABASE = 'rebuild_database', +} diff --git a/src/api-types/models/UpdateFileRequest.ts b/src/api-types/models/UpdateFileRequest.ts index b5bb188..224b018 100644 --- a/src/api-types/models/UpdateFileRequest.ts +++ b/src/api-types/models/UpdateFileRequest.ts @@ -10,5 +10,5 @@ export type UpdateFileRequest = { /** * Object with fields to update as keys and their new values */ - change_obj: Record; + change_obj: any; }; diff --git a/src/api-types/models/UpdateTaskDataRequest.ts b/src/api-types/models/UpdateTaskDataRequest.ts index c52509c..dbcd1c5 100644 --- a/src/api-types/models/UpdateTaskDataRequest.ts +++ b/src/api-types/models/UpdateTaskDataRequest.ts @@ -2,7 +2,9 @@ /* tslint:disable */ /* eslint-disable */ +import type { TaskType } from './TaskType'; + export type UpdateTaskDataRequest = { - task_key: string; - new_data: Record; + task_key: TaskType; + new_data: any; }; diff --git a/src/api-types/models/UpdateTaskOptionsRequest.ts b/src/api-types/models/UpdateTaskOptionsRequest.ts index 50ae9e8..2a465e0 100644 --- a/src/api-types/models/UpdateTaskOptionsRequest.ts +++ b/src/api-types/models/UpdateTaskOptionsRequest.ts @@ -2,7 +2,9 @@ /* tslint:disable */ /* eslint-disable */ +import type { TaskType } from './TaskType'; + export type UpdateTaskOptionsRequest = { - task_key: string; - new_options: Record; + task_key: TaskType; + new_options: any; }; diff --git a/src/api-types/models/UpdateTaskScheduleRequest.ts b/src/api-types/models/UpdateTaskScheduleRequest.ts index fb27cd8..6558b1a 100644 --- a/src/api-types/models/UpdateTaskScheduleRequest.ts +++ b/src/api-types/models/UpdateTaskScheduleRequest.ts @@ -3,8 +3,9 @@ /* eslint-disable */ import type { Schedule } from './Schedule'; +import type { TaskType } from './TaskType'; export type UpdateTaskScheduleRequest = { - task_key: string; + task_key: TaskType; new_schedule: Schedule; }; diff --git a/src/app/components/task-settings/task-settings.component.ts b/src/app/components/task-settings/task-settings.component.ts index f96b57e..c616177 100644 --- a/src/app/components/task-settings/task-settings.component.ts +++ b/src/app/components/task-settings/task-settings.component.ts @@ -1,6 +1,6 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA } from '@angular/material/dialog'; -import { Task } from 'api-types'; +import { Task, TaskType } from 'api-types'; import { PostsService } from 'app/posts.services'; @Component({ @@ -9,7 +9,7 @@ import { PostsService } from 'app/posts.services'; styleUrls: ['./task-settings.component.scss'] }) export class TaskSettingsComponent { - task_key: string; + task_key: TaskType; new_options = {}; task: Task = null; diff --git a/src/app/components/tasks/tasks.component.html b/src/app/components/tasks/tasks.component.html index 02d8fe1..1264d8d 100644 --- a/src/app/components/tasks/tasks.component.html +++ b/src/app/components/tasks/tasks.component.html @@ -49,7 +49,7 @@ Actions
-
+