You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2887 lines
97 KiB
JavaScript
2887 lines
97 KiB
JavaScript
const { v4: uuid } = require('uuid');
|
|
const fs = require('fs-extra');
|
|
const { promisify } = require('util');
|
|
const http = require('http');
|
|
const https = require('https');
|
|
const auth_api = require('./authentication/auth');
|
|
const oidc_api = require('./authentication/oidc');
|
|
const path = require('path');
|
|
const compression = require('compression');
|
|
const multer = require('multer');
|
|
const express = require("express");
|
|
const rateLimit = require('express-rate-limit');
|
|
const bodyParser = require("body-parser");
|
|
const archiver = require('archiver');
|
|
const unzipper = require('unzipper');
|
|
const db_api = require('./db');
|
|
const utils = require('./utils')
|
|
const low = require('./lowdb-compat')
|
|
const fetch = globalThis.fetch;
|
|
const URL = require('url').URL;
|
|
const CONSTS = require('./consts')
|
|
const read_last_lines = require('read-last-lines');
|
|
const ps = require('ps-node');
|
|
|
|
const logger = require('./logger');
|
|
const config_api = require('./config.js');
|
|
const downloader_api = require('./downloader');
|
|
const tasks_api = require('./tasks');
|
|
const subscriptions_api = require('./subscriptions');
|
|
const categories_api = require('./categories');
|
|
const twitch_api = require('./twitch');
|
|
const youtubedl_api = require('./youtube-dl');
|
|
const archive_api = require('./archive');
|
|
const files_api = require('./files');
|
|
const notifications_api = require('./notifications');
|
|
|
|
var app = express();
|
|
|
|
function parseTrustProxySetting(value) {
|
|
if (value === undefined || value === null) return undefined;
|
|
if (typeof value !== 'string') return value;
|
|
|
|
const trimmed = value.trim();
|
|
if (trimmed === '') return undefined;
|
|
|
|
const lowerValue = trimmed.toLowerCase();
|
|
if (lowerValue === 'true') return true;
|
|
if (lowerValue === 'false') return false;
|
|
if (/^\d+$/.test(trimmed)) return parseInt(trimmed, 10);
|
|
if (trimmed.includes(',')) return trimmed.split(',').map(item => item.trim()).filter(item => item !== '');
|
|
|
|
return trimmed;
|
|
}
|
|
|
|
function configureExpressTrustProxy() {
|
|
const trustProxyFromEnv = parseTrustProxySetting(process.env.YTDL_TRUST_PROXY);
|
|
if (trustProxyFromEnv !== undefined) {
|
|
app.set('trust proxy', trustProxyFromEnv);
|
|
logger.info(`Express trust proxy configured from YTDL_TRUST_PROXY: ${JSON.stringify(trustProxyFromEnv)}`);
|
|
return;
|
|
}
|
|
|
|
const reverseProxyWhitelist = config_api.getConfigItem('ytdl_reverse_proxy_whitelist');
|
|
if (reverseProxyWhitelist && reverseProxyWhitelist.trim() !== '') {
|
|
const trustedProxies = reverseProxyWhitelist
|
|
.split(',')
|
|
.map(item => item.trim())
|
|
.filter(item => item !== '');
|
|
|
|
if (trustedProxies.length > 0) {
|
|
app.set('trust proxy', trustedProxies);
|
|
logger.info('Express trust proxy configured from reverse proxy whitelist.');
|
|
}
|
|
}
|
|
}
|
|
|
|
// database setup
|
|
const FileSync = require('./lowdb-compat/adapters/FileSync');
|
|
|
|
const adapter = new FileSync('./appdata/db.json');
|
|
const db = low(adapter)
|
|
|
|
const users_adapter = new FileSync('./appdata/users.json');
|
|
const users_db = low(users_adapter);
|
|
|
|
// env var setup
|
|
|
|
const umask = process.env.YTDL_UMASK;
|
|
if (umask) process.umask(parseInt(umask));
|
|
|
|
// check if debug mode
|
|
let debugMode = process.env.YTDL_MODE === 'debug';
|
|
|
|
const admin_token = '4241b401-7236-493e-92b5-b72696b9d853';
|
|
|
|
// logging setup
|
|
|
|
config_api.initialize();
|
|
db_api.initialize(db, users_db);
|
|
auth_api.initialize(db_api);
|
|
|
|
// Set some defaults
|
|
db.defaults(
|
|
{
|
|
playlists: [],
|
|
files: [],
|
|
configWriteFlag: false,
|
|
downloads: {},
|
|
subscriptions: [],
|
|
files_to_db_migration_complete: false,
|
|
tasks_manager_role_migration_complete: false,
|
|
archives_migration_complete: false
|
|
}).write();
|
|
|
|
users_db.defaults(
|
|
{
|
|
users: [],
|
|
roles: {
|
|
"admin": {
|
|
"permissions": [
|
|
'filemanager',
|
|
'settings',
|
|
'subscriptions',
|
|
'sharing',
|
|
'advanced_download',
|
|
'downloads_manager'
|
|
]
|
|
}, "user": {
|
|
"permissions": [
|
|
'filemanager',
|
|
'subscriptions',
|
|
'sharing'
|
|
]
|
|
}
|
|
}
|
|
}
|
|
).write();
|
|
|
|
// config values
|
|
let url = null;
|
|
let backendPort = null;
|
|
let useDefaultDownloadingAgent = null;
|
|
let customDownloadingAgent = null;
|
|
let allowSubscriptions = null;
|
|
|
|
// other needed values
|
|
let url_domain = null;
|
|
let updaterStatus = null;
|
|
|
|
const concurrentStreams = {};
|
|
|
|
if (debugMode) logger.info('YTDL-Material in debug mode!');
|
|
|
|
// check if just updated
|
|
const just_updated = fs.existsSync('restart_update.json');
|
|
if (just_updated) {
|
|
updaterStatus = {
|
|
updating: false,
|
|
details: 'Update complete! You are now on ' + CONSTS['CURRENT_VERSION']
|
|
}
|
|
fs.unlinkSync('restart_update.json');
|
|
}
|
|
|
|
if (fs.existsSync('restart_general.json')) fs.unlinkSync('restart_general.json');
|
|
|
|
// updates & starts youtubedl (commented out b/c of repo takedown)
|
|
// startYoutubeDL();
|
|
|
|
var validDownloadingAgents = [
|
|
'aria2c',
|
|
'avconv',
|
|
'axel',
|
|
'curl',
|
|
'ffmpeg',
|
|
'httpie',
|
|
'wget'
|
|
];
|
|
|
|
const subscription_timeouts = {};
|
|
|
|
let version_info = null;
|
|
if (fs.existsSync('version.json')) {
|
|
version_info = fs.readJSONSync('version.json');
|
|
logger.verbose(`Version info: ${JSON.stringify(version_info, null, 2)}`);
|
|
} else {
|
|
version_info = {'type': 'N/A', 'tag': 'N/A', 'commit': 'N/A', 'date': 'N/A'};
|
|
}
|
|
|
|
// don't overwrite config if it already happened.. NOT
|
|
// let alreadyWritten = db.get('configWriteFlag').value();
|
|
|
|
// checks if config exists, if not, a config is auto generated
|
|
config_api.configExistsCheck();
|
|
|
|
setAndLoadConfig();
|
|
|
|
app.use(bodyParser.urlencoded({ extended: false }));
|
|
app.use(bodyParser.json());
|
|
|
|
// use passport
|
|
app.use(auth_api.passport.initialize());
|
|
|
|
// reverse proxy whitelist
|
|
app.use(reverseProxyWhitelistMiddleware);
|
|
|
|
// actual functions
|
|
|
|
async function checkMigrations() {
|
|
// 4.1->4.2 migration
|
|
|
|
const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value();
|
|
if (!simplified_db_migration_complete) {
|
|
logger.info('Beginning migration: 4.1->4.2+')
|
|
let success = await simplifyDBFileStructure();
|
|
success = success && await files_api.addMetadataPropertyToDB('view_count');
|
|
success = success && await files_api.addMetadataPropertyToDB('description');
|
|
success = success && await files_api.addMetadataPropertyToDB('height');
|
|
success = success && await files_api.addMetadataPropertyToDB('abr');
|
|
// sets migration to complete
|
|
db.set('simplified_db_migration_complete', true).write();
|
|
if (success) { logger.info('4.1->4.2+ migration complete!'); }
|
|
else { logger.error('Migration failed: 4.1->4.2+'); }
|
|
}
|
|
|
|
const new_db_system_migration_complete = db.get('new_db_system_migration_complete').value();
|
|
if (!new_db_system_migration_complete) {
|
|
logger.info('Beginning migration: 4.2->4.3+')
|
|
let success = await db_api.importJSONToDB(db.value(), users_db.value());
|
|
await tasks_api.setupTasks(); // necessary as tasks were not properly initialized at first
|
|
// sets migration to complete
|
|
db.set('new_db_system_migration_complete', true).write();
|
|
if (success) { logger.info('4.2->4.3+ migration complete!'); }
|
|
else { logger.error('Migration failed: 4.2->4.3+'); }
|
|
}
|
|
|
|
const tasks_manager_role_migration_complete = db.get('tasks_manager_role_migration_complete').value();
|
|
if (!tasks_manager_role_migration_complete) {
|
|
logger.info('Checking if tasks manager role permissions exist for admin user...');
|
|
const success = await auth_api.changeRolePermissions('admin', 'tasks_manager', 'yes');
|
|
if (success) logger.info('Task manager permissions check complete!');
|
|
else logger.error('Failed to auto add tasks manager permissions to admin role!');
|
|
db.set('tasks_manager_role_migration_complete', true).write();
|
|
}
|
|
|
|
const archives_migration_complete = db.get('archives_migration_complete').value();
|
|
if (!archives_migration_complete) {
|
|
logger.info('Checking if archives have been migrated...');
|
|
const imported_archives = await archive_api.importArchives();
|
|
if (imported_archives) logger.info('Archives migration complete!');
|
|
else logger.error('Failed to migrate archives!');
|
|
db.set('archives_migration_complete', true).write();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
async function simplifyDBFileStructure() {
|
|
// back up db files
|
|
const old_db_file = fs.readJSONSync('./appdata/db.json');
|
|
const old_users_db_file = fs.readJSONSync('./appdata/users.json');
|
|
fs.writeJSONSync('appdata/db.old.json', old_db_file);
|
|
fs.writeJSONSync('appdata/users.old.json', old_users_db_file);
|
|
|
|
// simplify
|
|
let users = users_db.get('users').value();
|
|
for (let i = 0; i < users.length; i++) {
|
|
const user = users[i];
|
|
if (user['files']['video'] !== undefined && user['files']['audio'] !== undefined) {
|
|
const user_files = user['files']['video'].concat(user['files']['audio']);
|
|
const user_db_path = users_db.get('users').find({uid: user['uid']});
|
|
user_db_path.assign({files: user_files}).write();
|
|
}
|
|
if (user['playlists']['video'] !== undefined && user['playlists']['audio'] !== undefined) {
|
|
const user_playlists = user['playlists']['video'].concat(user['playlists']['audio']);
|
|
const user_db_path = users_db.get('users').find({uid: user['uid']});
|
|
user_db_path.assign({playlists: user_playlists}).write();
|
|
}
|
|
}
|
|
|
|
if (db.get('files.video').value() !== undefined && db.get('files.audio').value() !== undefined) {
|
|
const files = db.get('files.video').value().concat(db.get('files.audio').value());
|
|
db.assign({files: files}).write();
|
|
}
|
|
|
|
if (db.get('playlists.video').value() !== undefined && db.get('playlists.audio').value() !== undefined) {
|
|
const playlists = db.get('playlists.video').value().concat(db.get('playlists.audio').value());
|
|
db.assign({playlists: playlists}).write();
|
|
}
|
|
|
|
|
|
return true;
|
|
}
|
|
|
|
// CIDR IP checking utility
|
|
function ipInCIDR(ip, cidr) {
|
|
const [range, bits = 32] = cidr.split('/');
|
|
const mask = ~(2 ** (32 - bits) - 1);
|
|
const ipNum = ip.split('.').reduce((int, oct) => (int << 8) + parseInt(oct, 10), 0) >>> 0;
|
|
const rangeNum = range.split('.').reduce((int, oct) => (int << 8) + parseInt(oct, 10), 0) >>> 0;
|
|
return (ipNum & mask) === (rangeNum & mask);
|
|
}
|
|
|
|
// Reverse proxy whitelist middleware
|
|
function reverseProxyWhitelistMiddleware(req, res, next) {
|
|
const whitelist = config_api.getConfigItem('ytdl_reverse_proxy_whitelist');
|
|
|
|
if (!whitelist || whitelist.trim() === '') {
|
|
// No whitelist configured, allow all
|
|
return next();
|
|
}
|
|
|
|
// Get the direct connecting IP (the reverse proxy itself, not the end client)
|
|
const proxyIp = (req.connection.remoteAddress || req.socket.remoteAddress || '').replace('::ffff:', '');
|
|
|
|
// Parse whitelist (can be comma-separated CIDRs)
|
|
const allowedRanges = whitelist.split(',').map(s => s.trim()).filter(s => s);
|
|
|
|
// Check if IP is in any of the allowed ranges
|
|
for (const range of allowedRanges) {
|
|
try {
|
|
if (ipInCIDR(proxyIp, range)) {
|
|
return next();
|
|
}
|
|
} catch (e) {
|
|
logger.warn(`Invalid CIDR range in whitelist: ${range}`);
|
|
}
|
|
}
|
|
|
|
logger.warn(`Access denied for reverse proxy IP ${proxyIp} - not in whitelist`);
|
|
return res.status(403).send('Access forbidden');
|
|
}
|
|
|
|
async function startServer() {
|
|
if (process.env.USING_HEROKU && process.env.PORT) {
|
|
// default to heroku port if using heroku
|
|
backendPort = process.env.PORT || backendPort;
|
|
|
|
// set config to port
|
|
await setPortItemFromENV();
|
|
}
|
|
|
|
// Check if SSL certificates are configured
|
|
const sslCertPath = config_api.getConfigItem('ytdl_ssl_cert_path');
|
|
const sslKeyPath = config_api.getConfigItem('ytdl_ssl_key_path');
|
|
|
|
let server;
|
|
|
|
if (sslCertPath && sslKeyPath && fs.existsSync(sslCertPath) && fs.existsSync(sslKeyPath)) {
|
|
// Start HTTPS server
|
|
const httpsOptions = {
|
|
cert: fs.readFileSync(sslCertPath),
|
|
key: fs.readFileSync(sslKeyPath)
|
|
};
|
|
|
|
server = https.createServer(httpsOptions, app);
|
|
server.listen(backendPort, function() {
|
|
logger.info(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on HTTPS PORT ${backendPort}`);
|
|
});
|
|
} else {
|
|
// Start HTTP server
|
|
if (sslCertPath || sslKeyPath) {
|
|
logger.warn('SSL certificate or key path configured but files not found. Starting HTTP server instead.');
|
|
}
|
|
|
|
server = http.createServer(app);
|
|
server.listen(backendPort, function() {
|
|
logger.info(`YoutubeDL-Material ${CONSTS['CURRENT_VERSION']} started on HTTP PORT ${backendPort}`);
|
|
});
|
|
}
|
|
}
|
|
|
|
async function updateServer(tag) {
|
|
// no tag provided means update to the latest version
|
|
if (!tag) {
|
|
const new_version_available = await isNewVersionAvailable();
|
|
if (!new_version_available) {
|
|
logger.error('ERROR: Failed to update - no update is available.');
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return new Promise(async resolve => {
|
|
// backup current dir
|
|
updaterStatus = {
|
|
updating: true,
|
|
'details': 'Backing up key server files...'
|
|
}
|
|
let backup_succeeded = await backupServerLite();
|
|
if (!backup_succeeded) {
|
|
resolve(false);
|
|
return false;
|
|
}
|
|
|
|
updaterStatus = {
|
|
updating: true,
|
|
'details': 'Downloading requested release...'
|
|
}
|
|
// grab new package.json and public folder
|
|
await downloadReleaseFiles(tag);
|
|
|
|
updaterStatus = {
|
|
updating: true,
|
|
'details': 'Installing new dependencies...'
|
|
}
|
|
// run npm install
|
|
await installDependencies();
|
|
|
|
updaterStatus = {
|
|
updating: true,
|
|
'details': 'Update complete! Restarting server...'
|
|
}
|
|
utils.restartServer(true);
|
|
}, err => {
|
|
logger.error(err);
|
|
updaterStatus = {
|
|
updating: false,
|
|
error: true,
|
|
'details': 'Update failed. Check error logs for more info.'
|
|
}
|
|
});
|
|
}
|
|
|
|
async function downloadReleaseFiles(tag) {
|
|
tag = tag ? tag : await getLatestVersion();
|
|
const safeTag = getValidatedReleaseTag(tag);
|
|
const releaseZipPath = getSafeReleaseZipPath(tag);
|
|
if (!safeTag || !releaseZipPath) {
|
|
logger.error(`Refusing to install release with invalid tag: ${tag}`);
|
|
return false;
|
|
}
|
|
|
|
return new Promise(async resolve => {
|
|
logger.info('Downloading new files...')
|
|
|
|
// downloads the latest release zip file
|
|
const zipDownloaded = await downloadReleaseZip(safeTag);
|
|
if (!zipDownloaded) {
|
|
resolve(false);
|
|
return;
|
|
}
|
|
|
|
// deletes contents of public dir
|
|
fs.removeSync(path.join(__dirname, 'public'));
|
|
fs.mkdirSync(path.join(__dirname, 'public'));
|
|
|
|
let replace_ignore_list = ['youtubedl-material/appdata/default.json',
|
|
'youtubedl-material/appdata/db.json',
|
|
'youtubedl-material/appdata/users.json',
|
|
'youtubedl-material/appdata/*']
|
|
logger.info(`Installing update ${safeTag}...`)
|
|
|
|
// downloads new package.json and adds new public dir files from the downloaded zip
|
|
fs.createReadStream(releaseZipPath).pipe(unzipper.Parse())
|
|
.on('entry', function (entry) {
|
|
var fileName = entry.path;
|
|
var is_dir = fileName.substring(fileName.length-1, fileName.length) === '/'
|
|
if (!is_dir && fileName.includes('youtubedl-material/public/')) {
|
|
// get public folder files
|
|
const actualFileName = fileName.replace('youtubedl-material/public/', '');
|
|
if (actualFileName.length !== 0 && actualFileName.substring(actualFileName.length-1, actualFileName.length) !== '/') {
|
|
const publicBasePath = path.join(__dirname, 'public');
|
|
const targetPublicPath = path.resolve(publicBasePath, actualFileName);
|
|
const relativePublicPath = path.relative(publicBasePath, targetPublicPath);
|
|
if (relativePublicPath.startsWith('..') || path.isAbsolute(relativePublicPath)) {
|
|
logger.warn(`Skipping unsafe public file path during update extraction: ${actualFileName}`);
|
|
entry.autodrain();
|
|
return;
|
|
}
|
|
|
|
fs.ensureDirSync(path.dirname(targetPublicPath));
|
|
entry.pipe(fs.createWriteStream(targetPublicPath));
|
|
} else {
|
|
entry.autodrain();
|
|
}
|
|
} else if (!is_dir && !replace_ignore_list.includes(fileName)) {
|
|
// get package.json
|
|
const actualFileName = fileName.replace('youtubedl-material/', '');
|
|
const repoBasePath = path.resolve(__dirname);
|
|
const targetFilePath = path.resolve(repoBasePath, actualFileName);
|
|
const relativeRepoPath = path.relative(repoBasePath, targetFilePath);
|
|
if (relativeRepoPath.startsWith('..') || path.isAbsolute(relativeRepoPath)) {
|
|
logger.warn(`Skipping unsafe file path during update extraction: ${actualFileName}`);
|
|
entry.autodrain();
|
|
return;
|
|
}
|
|
logger.verbose('Downloading file ' + actualFileName);
|
|
entry.pipe(fs.createWriteStream(targetFilePath));
|
|
} else {
|
|
entry.autodrain();
|
|
}
|
|
})
|
|
.on('close', function () {
|
|
resolve(true);
|
|
});
|
|
});
|
|
}
|
|
|
|
async function downloadReleaseZip(tag) {
|
|
return new Promise(async resolve => {
|
|
const safeTag = getValidatedReleaseTag(tag);
|
|
const resolvedOutputPath = getSafeReleaseZipPath(tag);
|
|
if (!safeTag || !resolvedOutputPath) {
|
|
logger.error(`Refusing to download release with invalid tag: ${tag}`);
|
|
resolve(false);
|
|
return;
|
|
}
|
|
|
|
// get name of zip file, which depends on the version
|
|
const tag_without_v = safeTag.substring(1, safeTag.length);
|
|
const zip_file_name = `youtubedl-material-${tag_without_v}.zip`;
|
|
const latest_zip_link = `https://github.com/voc0der/YoutubeDL-Material/releases/download/${encodeURIComponent(safeTag)}/${encodeURIComponent(zip_file_name)}`;
|
|
|
|
// download zip from release
|
|
const res = await fetch(latest_zip_link);
|
|
if (!res.ok) {
|
|
logger.error(`Failed to download release zip for ${safeTag}: HTTP ${res.status}`);
|
|
resolve(false);
|
|
return;
|
|
}
|
|
await utils.writeFetchResponseToFile(res, fs.createWriteStream(resolvedOutputPath), 'update ' + safeTag);
|
|
resolve(true);
|
|
});
|
|
|
|
}
|
|
|
|
async function installDependencies() {
|
|
var child_process = require('child_process');
|
|
var exec = promisify(child_process.exec);
|
|
|
|
await exec('npm install',{stdio:[0,1,2]});
|
|
return true;
|
|
}
|
|
|
|
async function backupServerLite() {
|
|
await fs.ensureDir(path.join(__dirname, 'appdata', 'backups'));
|
|
let output_path = path.join('appdata', 'backups', `backup-${Date.now()}.zip`);
|
|
logger.info(`Backing up your non-video/audio files to ${output_path}. This may take up to a few seconds/minutes.`);
|
|
let output = fs.createWriteStream(path.join(__dirname, output_path));
|
|
|
|
await new Promise(resolve => {
|
|
var archive = archiver('zip', {
|
|
gzip: true,
|
|
zlib: { level: 9 } // Sets the compression level.
|
|
});
|
|
|
|
archive.on('error', function(err) {
|
|
logger.error(err);
|
|
resolve(false);
|
|
});
|
|
|
|
// pipe archive data to the output file
|
|
archive.pipe(output);
|
|
|
|
// ignore certain directories (ones with video or audio files)
|
|
const files_to_ignore = [path.join(config_api.getConfigItem('ytdl_subscriptions_base_path'), '**'),
|
|
path.join(config_api.getConfigItem('ytdl_audio_folder_path'), '**'),
|
|
path.join(config_api.getConfigItem('ytdl_video_folder_path'), '**'),
|
|
'appdata/backups/backup-*.zip'];
|
|
|
|
archive.glob('**/*', {
|
|
ignore: files_to_ignore
|
|
});
|
|
|
|
resolve(archive.finalize());
|
|
});
|
|
|
|
// wait a tiny bit for the zip to reload in fs
|
|
await utils.wait(100);
|
|
return true;
|
|
}
|
|
|
|
async function isNewVersionAvailable() {
|
|
// gets tag of the latest version of youtubedl-material, compare to current version
|
|
const latest_tag = await getLatestVersion();
|
|
const current_tag = CONSTS['CURRENT_VERSION'];
|
|
if (latest_tag > current_tag) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function getLatestVersion() {
|
|
const res = await fetch('https://api.github.com/repos/voc0der/youtubedl-material/releases/latest', {method: 'Get'});
|
|
const json = await res.json();
|
|
|
|
if (json['message']) {
|
|
// means there's an error in getting latest version
|
|
logger.error(`ERROR: Received the following message from GitHub's API:`);
|
|
logger.error(json['message']);
|
|
if (json['documentation_url']) logger.error(`Associated URL: ${json['documentation_url']}`)
|
|
}
|
|
return json['tag_name'];
|
|
}
|
|
|
|
async function killAllDownloads() {
|
|
const lookupAsync = promisify(ps.lookup);
|
|
let resultList = null;
|
|
|
|
try {
|
|
resultList = await lookupAsync({
|
|
command: 'youtube-dl'
|
|
});
|
|
} catch (err) {
|
|
// failed to get list of processes
|
|
logger.error('Failed to get a list of running youtube-dl processes.');
|
|
logger.error(err);
|
|
return {
|
|
details: err,
|
|
success: false
|
|
};
|
|
}
|
|
|
|
// processes that contain the string 'youtube-dl' in the name will be looped
|
|
resultList.forEach(function( process ){
|
|
if (process) {
|
|
ps.kill(process.pid, 'SIGKILL', function( err ) {
|
|
if (err) {
|
|
// failed to kill, process may have ended on its own
|
|
logger.warn(`Failed to kill process with PID ${process.pid}`);
|
|
logger.warn(err);
|
|
}
|
|
else {
|
|
logger.verbose(`Process ${process.pid} has been killed!`);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
|
|
return {
|
|
success: true
|
|
};
|
|
}
|
|
|
|
async function setPortItemFromENV() {
|
|
config_api.setConfigItem('ytdl_port', backendPort.toString());
|
|
await utils.wait(100);
|
|
return true;
|
|
}
|
|
|
|
function getOIDCMigrateTargetFromEnv() {
|
|
return process.env.ytdl_oidc_migrate_videos || process.env.YTDL_OIDC_MIGRATE_VIDEOS || null;
|
|
}
|
|
|
|
function getScopedFilterByUser(user_uid) {
|
|
if (!config_api.getConfigItem('ytdl_multi_user_mode')) return {};
|
|
return {user_uid: user_uid};
|
|
}
|
|
|
|
async function migrateUnassignedVideosToConfiguredUser() {
|
|
const migrate_target = getOIDCMigrateTargetFromEnv();
|
|
if (!migrate_target) return true;
|
|
|
|
const normalized_target = String(migrate_target).trim();
|
|
const safe_uid = auth_api.sanitizeUserUID(normalized_target);
|
|
if (!safe_uid) {
|
|
throw new Error(`Invalid ytdl_oidc_migrate_videos value '${normalized_target}'. It must be a valid uid.`);
|
|
}
|
|
|
|
let target_user = await db_api.getRecord('users', {uid: safe_uid});
|
|
if (!target_user) {
|
|
target_user = await db_api.getRecord('users', {name: normalized_target});
|
|
}
|
|
|
|
if (!target_user) {
|
|
throw new Error(`ytdl_oidc_migrate_videos requested migration to '${normalized_target}', but no such user exists.`);
|
|
}
|
|
|
|
const unassigned_filter = {user_uid: null};
|
|
const unassigned_file_count = await db_api.getRecords('files', unassigned_filter, true);
|
|
const unassigned_playlist_count = await db_api.getRecords('playlists', unassigned_filter, true);
|
|
|
|
if (!unassigned_file_count && !unassigned_playlist_count) {
|
|
throw new Error('No unassigned videos/files or playlists exist for ytdl_oidc_migrate_videos. Remove this env variable to continue startup.');
|
|
}
|
|
|
|
if (unassigned_file_count) {
|
|
const migrated_files = await db_api.updateRecords('files', unassigned_filter, {user_uid: target_user.uid});
|
|
if (!migrated_files) {
|
|
throw new Error(`Failed to migrate unassigned videos/files to user '${target_user.uid}'.`);
|
|
}
|
|
}
|
|
|
|
if (unassigned_playlist_count) {
|
|
const migrated_playlists = await db_api.updateRecords('playlists', unassigned_filter, {user_uid: target_user.uid});
|
|
if (!migrated_playlists) {
|
|
throw new Error(`Failed to migrate unassigned playlists to user '${target_user.uid}'.`);
|
|
}
|
|
}
|
|
|
|
logger.info(`Migrated ${unassigned_file_count} unassigned videos/files and ${unassigned_playlist_count} unassigned playlists to user '${target_user.uid}' from ytdl_oidc_migrate_videos.`);
|
|
return true;
|
|
}
|
|
|
|
async function setAndLoadConfig() {
|
|
try {
|
|
await setConfigFromEnv();
|
|
await loadConfig();
|
|
} catch (err) {
|
|
logger.error(`Startup failed: ${err.message}`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
async function setConfigFromEnv() {
|
|
const config_items = getEnvConfigItems();
|
|
if (!config_items || config_items.length === 0) return true;
|
|
const success = config_api.setConfigItems(config_items);
|
|
if (success) {
|
|
logger.info('Config items set using ENV variables.');
|
|
await utils.wait(100);
|
|
return true;
|
|
} else {
|
|
logger.error('ERROR: Failed to set config items using ENV variables.');
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function loadConfig() {
|
|
loadConfigValues();
|
|
|
|
const oidc_enabled = oidc_api.isEnabled();
|
|
if (oidc_enabled && !config_api.getConfigItem('ytdl_multi_user_mode')) {
|
|
logger.error('OIDC startup failed: multi-user mode must be enabled when OIDC is enabled.');
|
|
process.exit(1);
|
|
}
|
|
|
|
try {
|
|
await oidc_api.initialize();
|
|
} catch (err) {
|
|
logger.error(`OIDC startup failed: ${err.message}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
// connect to DB
|
|
if (!config_api.getConfigItem('ytdl_use_local_db'))
|
|
await db_api.connectToDB();
|
|
db_api.database_initialized = true;
|
|
db_api.database_initialized_bs.next(true);
|
|
|
|
// check migrations
|
|
await checkMigrations();
|
|
await migrateUnassignedVideosToConfiguredUser();
|
|
|
|
// now this is done here due to youtube-dl's repo takedown
|
|
await startYoutubeDL();
|
|
|
|
// get subscriptions
|
|
if (allowSubscriptions) {
|
|
// set downloading to false
|
|
let subscriptions = await subscriptions_api.getAllSubscriptions();
|
|
subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
|
|
subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false, child_process: null});
|
|
// runs initially, then runs every ${subscriptionCheckInterval} seconds
|
|
subscriptions_api.watchSubscriptionsInterval();
|
|
}
|
|
|
|
// start the server here
|
|
startServer();
|
|
|
|
return true;
|
|
}
|
|
|
|
function loadConfigValues() {
|
|
url = !debugMode ? config_api.getConfigItem('ytdl_url') : 'http://localhost:4200';
|
|
backendPort = config_api.getConfigItem('ytdl_port');
|
|
useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
|
|
customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
|
|
allowSubscriptions = config_api.getConfigItem('ytdl_allow_subscriptions');
|
|
|
|
if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) {
|
|
logger.info(`Using non-default downloading agent \'${customDownloadingAgent}\'`)
|
|
} else {
|
|
customDownloadingAgent = null;
|
|
}
|
|
|
|
// empty url defaults to default URL
|
|
if (!url || url === '') url = 'http://example.com'
|
|
url_domain = new URL(url);
|
|
|
|
let logger_level = config_api.getConfigItem('ytdl_logger_level');
|
|
utils.updateLoggerLevel(logger_level);
|
|
|
|
configureExpressTrustProxy();
|
|
}
|
|
|
|
function getOrigin() {
|
|
if (process.env.CODESPACES) return `https://${process.env.CODESPACE_NAME}-4200.${process.env.GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}`;
|
|
return url_domain.origin;
|
|
}
|
|
|
|
const VALID_RELEASE_TAG_PATTERN = /^v[0-9A-Za-z][0-9A-Za-z._-]*$/;
|
|
const XML_ENTITY_MAP = {
|
|
'&': '&',
|
|
'<': '<',
|
|
'>': '>',
|
|
'"': '"',
|
|
"'": '''
|
|
};
|
|
|
|
function getValidatedReleaseTag(tag) {
|
|
return (typeof tag === 'string' && VALID_RELEASE_TAG_PATTERN.test(tag)) ? tag : null;
|
|
}
|
|
|
|
function getSafeReleaseZipPath(tag) {
|
|
const validTag = getValidatedReleaseTag(tag);
|
|
if (!validTag) return null;
|
|
|
|
const resolvedOutputPath = path.resolve(__dirname, `youtubedl-material-release-${validTag}.zip`);
|
|
const relativeOutputPath = path.relative(__dirname, resolvedOutputPath);
|
|
if (relativeOutputPath.startsWith('..') || path.isAbsolute(relativeOutputPath)) return null;
|
|
|
|
return resolvedOutputPath;
|
|
}
|
|
|
|
function escapeXmlEntities(value) {
|
|
if (value === undefined || value === null) return value;
|
|
return String(value).replace(/[&<>"']/g, char => XML_ENTITY_MAP[char]);
|
|
}
|
|
|
|
// gets a list of config items that are stored as an environment variable
|
|
function getEnvConfigItems() {
|
|
let config_items = [];
|
|
|
|
let config_item_keys = Object.keys(config_api.CONFIG_ITEMS);
|
|
for (let i = 0; i < config_item_keys.length; i++) {
|
|
let key = config_item_keys[i];
|
|
if (process['env'][key] || process['env'][key.toUpperCase()]) {
|
|
const config_item = generateEnvVarConfigItem(key);
|
|
config_items.push(config_item);
|
|
}
|
|
}
|
|
|
|
return config_items;
|
|
}
|
|
|
|
// gets value of a config item and stores it in an object
|
|
function generateEnvVarConfigItem(key) {
|
|
return {key: key, value: process['env'][key] || process['env'][key.toUpperCase()]};
|
|
}
|
|
|
|
// youtube-dl functions
|
|
|
|
async function startYoutubeDL() {
|
|
// auto update youtube-dl
|
|
await youtubedl_api.checkForYoutubeDLUpdate();
|
|
}
|
|
|
|
app.use(function(req, res, next) {
|
|
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization");
|
|
res.header("Access-Control-Allow-Origin", getOrigin());
|
|
if (req.method === 'OPTIONS') {
|
|
res.sendStatus(200);
|
|
} else {
|
|
next();
|
|
}
|
|
});
|
|
|
|
app.use(function(req, res, next) {
|
|
if (!req.path.includes('/api/')) {
|
|
next();
|
|
} else if (req.path.includes('/api/auth/oidc/login') ||
|
|
req.path.includes('/api/auth/oidc/callback') ||
|
|
req.path.includes('/api/auth/oidc/status')) {
|
|
next();
|
|
} else if (req.query.apiKey === admin_token) {
|
|
next();
|
|
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
|
|
next();
|
|
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss') || req.path.includes('/api/telegramRequest')) {
|
|
next();
|
|
} else {
|
|
logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key present: ${!!req.query.apiKey}`);
|
|
req.socket.end();
|
|
}
|
|
});
|
|
|
|
app.use(compression());
|
|
|
|
const rateLimitValidateOptions = {
|
|
xForwardedForHeader: false
|
|
};
|
|
|
|
const testCookiesRateLimiter = rateLimit({
|
|
windowMs: 60 * 1000,
|
|
max: 10,
|
|
standardHeaders: true,
|
|
legacyHeaders: false,
|
|
validate: rateLimitValidateOptions,
|
|
message: {
|
|
success: false,
|
|
error: 'Too many cookie test requests. Please wait a minute and try again.'
|
|
}
|
|
});
|
|
|
|
const apiRateLimiter = rateLimit({
|
|
windowMs: 60 * 1000,
|
|
max: 300,
|
|
standardHeaders: true,
|
|
legacyHeaders: false,
|
|
validate: rateLimitValidateOptions,
|
|
// Keep public media/feed endpoints usable while protecting stateful/file-system routes.
|
|
skip: (req) => req.path.includes('/api/stream/') ||
|
|
req.path.includes('/api/thumbnail/') ||
|
|
req.path.includes('/api/rss') ||
|
|
req.path.includes('/api/telegramRequest')
|
|
});
|
|
|
|
const authRateLimiter = rateLimit({
|
|
windowMs: 15 * 60 * 1000,
|
|
max: 25,
|
|
standardHeaders: true,
|
|
legacyHeaders: false,
|
|
validate: rateLimitValidateOptions,
|
|
message: {
|
|
success: false,
|
|
error: 'Too many authentication requests. Please wait and try again.'
|
|
}
|
|
});
|
|
|
|
app.use('/api', apiRateLimiter);
|
|
app.use('/api/auth', authRateLimiter);
|
|
|
|
function isPublicAuthPath(req_path) {
|
|
return req_path.includes('/api/auth/register')
|
|
|| req_path.includes('/api/auth/oidc/login')
|
|
|| req_path.includes('/api/auth/oidc/callback')
|
|
|| req_path.includes('/api/auth/oidc/status');
|
|
}
|
|
|
|
const optionalJwt = async function (req, res, next) {
|
|
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
|
|
if (multiUserMode && ((req.body && req.body.uuid) || (req.query && req.query.uuid)) && (req.path.includes('/api/getFile') ||
|
|
req.path.includes('/api/stream') ||
|
|
req.path.includes('/api/getPlaylist') ||
|
|
req.path.includes('/api/downloadFileFromServer'))) {
|
|
// check if shared video
|
|
const using_body = req.body && req.body.uuid;
|
|
const uuid = using_body ? req.body.uuid : req.query.uuid;
|
|
const uid = using_body ? req.body.uid : req.query.uid;
|
|
const playlist_id = using_body ? req.body.playlist_id : req.query.playlist_id;
|
|
const file = !playlist_id ? await auth_api.getUserVideo(uuid, uid, true) : await files_api.getPlaylist(playlist_id, uuid, true);
|
|
if (file) {
|
|
req.can_watch = true;
|
|
return next();
|
|
} else {
|
|
res.sendStatus(401);
|
|
return;
|
|
}
|
|
} else if (multiUserMode && !isPublicAuthPath(req.path)) {
|
|
if (!req.query.jwt) {
|
|
res.sendStatus(401);
|
|
return;
|
|
}
|
|
return auth_api.passport.authenticate('jwt', { session: false })(req, res, next);
|
|
}
|
|
return next();
|
|
};
|
|
|
|
app.get('/api/config', function(req, res) {
|
|
let config_file = config_api.getConfigFile();
|
|
res.send({
|
|
config_file: config_file,
|
|
success: !!config_file
|
|
});
|
|
});
|
|
|
|
app.post('/api/setConfig', optionalJwt, function(req, res) {
|
|
let new_config_file = req.body.new_config_file;
|
|
if (new_config_file && new_config_file['YoutubeDLMaterial']) {
|
|
let success = config_api.setConfigFile(new_config_file);
|
|
loadConfigValues(); // reloads config values that exist as variables
|
|
res.send({
|
|
success: success
|
|
});
|
|
} else {
|
|
logger.error('Tried to save invalid config file!')
|
|
res.sendStatus(400);
|
|
}
|
|
});
|
|
|
|
app.get('/api/versionInfo', (req, res) => {
|
|
res.send({version_info: version_info});
|
|
});
|
|
|
|
app.post('/api/restartServer', optionalJwt, (req, res) => {
|
|
// delayed by a little bit so that the client gets a response
|
|
setTimeout(() => {utils.restartServer()}, 100);
|
|
res.send({success: true});
|
|
});
|
|
|
|
app.get('/api/getDBInfo', optionalJwt, async (req, res) => {
|
|
const db_info = await db_api.getDBStats();
|
|
res.send(db_info);
|
|
});
|
|
|
|
app.post('/api/transferDB', optionalJwt, async (req, res) => {
|
|
const local_to_remote = req.body.local_to_remote;
|
|
let success = null;
|
|
let error = '';
|
|
if (local_to_remote === config_api.getConfigItem('ytdl_use_local_db')) {
|
|
success = await db_api.transferDB(local_to_remote);
|
|
if (!success) error = 'Unknown error';
|
|
else config_api.setConfigItem('ytdl_use_local_db', !local_to_remote);
|
|
} else {
|
|
success = false;
|
|
error = `Failed to transfer DB as it cannot transition into its current status: ${local_to_remote ? 'MongoDB' : 'Local DB'}`;
|
|
logger.error(error);
|
|
}
|
|
|
|
res.send({success: success, error: error});
|
|
});
|
|
|
|
app.post('/api/testConnectionString', optionalJwt, async (req, res) => {
|
|
const connection_string = req.body.connection_string;
|
|
let success = null;
|
|
let error = '';
|
|
success = await db_api.connectToDB(0, true, connection_string);
|
|
if (!success) error = 'Connection string failed.';
|
|
|
|
res.send({success: success, error: error});
|
|
});
|
|
|
|
app.post('/api/downloadFile', optionalJwt, async function(req, res) {
|
|
req.setTimeout(0); // remove timeout in case of long videos
|
|
const url = req.body.url;
|
|
const type = req.body.type ? req.body.type : 'video';
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const options = {
|
|
customArgs: req.body.customArgs,
|
|
additionalArgs: req.body.additionalArgs,
|
|
customOutput: req.body.customOutput,
|
|
selectedHeight: req.body.selectedHeight,
|
|
maxHeight: req.body.maxHeight,
|
|
customQualityConfiguration: req.body.customQualityConfiguration,
|
|
youtubeUsername: req.body.youtubeUsername,
|
|
youtubePassword: req.body.youtubePassword,
|
|
ui_uid: req.body.ui_uid,
|
|
cropFileSettings: req.body.cropFileSettings,
|
|
ignoreArchive: req.body.ignoreArchive
|
|
};
|
|
|
|
const download = await downloader_api.createDownload(url, type, options, user_uid);
|
|
|
|
if (download) {
|
|
res.send({download: download});
|
|
} else {
|
|
res.sendStatus(500);
|
|
}
|
|
});
|
|
|
|
app.post('/api/killAllDownloads', optionalJwt, async function(req, res) {
|
|
const result_obj = await killAllDownloads();
|
|
res.send(result_obj);
|
|
});
|
|
|
|
app.post('/api/generateArgs', optionalJwt, async function(req, res) {
|
|
const url = req.body.url;
|
|
const type = req.body.type;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const options = {
|
|
customArgs: req.body.customArgs,
|
|
additionalArgs: req.body.additionalArgs,
|
|
customOutput: req.body.customOutput,
|
|
selectedHeight: req.body.selectedHeight,
|
|
maxHeight: req.body.maxHeight,
|
|
customQualityConfiguration: req.body.customQualityConfiguration,
|
|
youtubeUsername: req.body.youtubeUsername,
|
|
youtubePassword: req.body.youtubePassword,
|
|
ui_uid: req.body.ui_uid,
|
|
cropFileSettings: req.body.cropFileSettings
|
|
};
|
|
|
|
const args = await downloader_api.generateArgs(url, type, options, user_uid, true);
|
|
res.send({args: args});
|
|
});
|
|
|
|
// gets all download mp3s
|
|
app.get('/api/getMp3s', optionalJwt, async function(req, res) {
|
|
// TODO: simplify
|
|
let mp3s = await db_api.getRecords('files', {isAudio: true});
|
|
let playlists = await db_api.getRecords('playlists');
|
|
const is_authenticated = req.isAuthenticated();
|
|
if (is_authenticated) {
|
|
// get user audio files/playlists
|
|
auth_api.passport.authenticate('jwt')
|
|
mp3s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: true});
|
|
playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove?
|
|
}
|
|
|
|
mp3s = JSON.parse(JSON.stringify(mp3s));
|
|
|
|
res.send({
|
|
mp3s: mp3s,
|
|
playlists: playlists
|
|
});
|
|
});
|
|
|
|
// gets all download mp4s
|
|
app.get('/api/getMp4s', optionalJwt, async function(req, res) {
|
|
let mp4s = await db_api.getRecords('files', {isAudio: false});
|
|
let playlists = await db_api.getRecords('playlists');
|
|
|
|
const is_authenticated = req.isAuthenticated();
|
|
if (is_authenticated) {
|
|
// get user videos/playlists
|
|
auth_api.passport.authenticate('jwt')
|
|
mp4s = await db_api.getRecords('files', {user_uid: req.user.uid, isAudio: false});
|
|
playlists = await db_api.getRecords('playlists', {user_uid: req.user.uid}); // TODO: remove?
|
|
}
|
|
|
|
mp4s = JSON.parse(JSON.stringify(mp4s));
|
|
|
|
res.send({
|
|
mp4s: mp4s,
|
|
playlists: playlists
|
|
});
|
|
});
|
|
|
|
app.post('/api/getFile', optionalJwt, async function (req, res) {
|
|
const uid = req.body.uid;
|
|
const uuid = req.body.uuid;
|
|
let file = null;
|
|
if (req.isAuthenticated()) {
|
|
file = await files_api.getVideo(uid, req.user.uid);
|
|
} else if (uuid) {
|
|
file = await auth_api.getUserVideo(uuid, uid, true);
|
|
} else {
|
|
file = await files_api.getVideo(uid);
|
|
}
|
|
|
|
// check if chat exists for twitch videos
|
|
if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
|
|
|
|
if (file) {
|
|
res.send({
|
|
success: true,
|
|
file: file
|
|
});
|
|
} else {
|
|
res.send({
|
|
success: false
|
|
});
|
|
}
|
|
});
|
|
|
|
app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
|
|
// these are returned
|
|
const sort = req.body.sort;
|
|
const range = req.body.range;
|
|
const text_search = req.body.text_search;
|
|
const file_type_filter = req.body.file_type_filter;
|
|
const favorite_filter = req.body.favorite_filter;
|
|
const sub_id = req.body.sub_id;
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const {files, file_count} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
|
|
|
|
res.send({
|
|
files: files,
|
|
file_count: file_count,
|
|
});
|
|
});
|
|
|
|
app.post('/api/updateFile', optionalJwt, async function (req, res) {
|
|
const uid = req.body.uid;
|
|
const change_obj = req.body.change_obj;
|
|
|
|
const file = await db_api.updateRecord('files', {uid: uid}, change_obj);
|
|
|
|
if (!file) {
|
|
res.send({
|
|
success: false,
|
|
error: 'File could not be found'
|
|
});
|
|
} else {
|
|
res.send({
|
|
success: true
|
|
});
|
|
}
|
|
});
|
|
|
|
app.post('/api/checkConcurrentStream', async (req, res) => {
|
|
const uid = req.body.uid;
|
|
|
|
const DEAD_SERVER_THRESHOLD = 10;
|
|
|
|
if (concurrentStreams[uid] && Date.now()/1000 - concurrentStreams[uid]['unix_timestamp'] > DEAD_SERVER_THRESHOLD) {
|
|
logger.verbose( `Killing dead stream on ${uid}`);
|
|
delete concurrentStreams[uid];
|
|
}
|
|
|
|
res.send({stream: concurrentStreams[uid]})
|
|
});
|
|
|
|
app.post('/api/updateConcurrentStream', optionalJwt, async (req, res) => {
|
|
const uid = req.body.uid;
|
|
const playback_timestamp = req.body.playback_timestamp;
|
|
const unix_timestamp = req.body.unix_timestamp;
|
|
const playing = req.body.playing;
|
|
|
|
concurrentStreams[uid] = {
|
|
playback_timestamp: playback_timestamp,
|
|
unix_timestamp: unix_timestamp,
|
|
playing: playing
|
|
}
|
|
|
|
res.send({stream: concurrentStreams[uid]})
|
|
});
|
|
|
|
app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => {
|
|
var id = req.body.id;
|
|
var type = req.body.type;
|
|
var uuid = req.body.uuid;
|
|
var sub = req.body.sub;
|
|
var user_uid = null;
|
|
|
|
if (req.isAuthenticated()) {
|
|
user_uid = req.user.uid;
|
|
uuid = req.user.uid;
|
|
}
|
|
|
|
const chat_file = await twitch_api.getTwitchChatByFileID(id, type, user_uid, uuid, sub);
|
|
|
|
res.send({
|
|
chat: chat_file
|
|
});
|
|
});
|
|
|
|
app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
|
|
var id = req.body.id;
|
|
var type = req.body.type;
|
|
var vodId = req.body.vodId;
|
|
var uuid = req.body.uuid;
|
|
var sub = req.body.sub;
|
|
var user_uid = null;
|
|
|
|
if (req.isAuthenticated()) {
|
|
user_uid = req.user.uid;
|
|
uuid = req.user.uid;
|
|
}
|
|
|
|
// check if file already exists. if so, send that instead
|
|
const file_exists_check = await twitch_api.getTwitchChatByFileID(id, type, user_uid, uuid, sub);
|
|
if (file_exists_check) {
|
|
res.send({chat: file_exists_check});
|
|
return;
|
|
}
|
|
|
|
const full_chat = await twitch_api.downloadTwitchChatByVODID(vodId, id, type, user_uid, sub);
|
|
|
|
res.send({
|
|
chat: full_chat
|
|
});
|
|
});
|
|
|
|
// video sharing
|
|
app.post('/api/enableSharing', optionalJwt, async (req, res) => {
|
|
var uid = req.body.uid;
|
|
var is_playlist = req.body.is_playlist;
|
|
let success = false;
|
|
// multi-user mode
|
|
if (req.isAuthenticated()) {
|
|
// if multi user mode, use this method instead
|
|
success = auth_api.changeSharingMode(req.user.uid, uid, is_playlist, true);
|
|
res.send({success: success});
|
|
return;
|
|
}
|
|
|
|
// single-user mode
|
|
try {
|
|
success = true;
|
|
if (!is_playlist) {
|
|
await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: true})
|
|
} else if (is_playlist) {
|
|
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: true});
|
|
} else if (false) {
|
|
// TODO: Implement.
|
|
} else {
|
|
// error
|
|
success = false;
|
|
}
|
|
|
|
} catch(err) {
|
|
logger.error(err);
|
|
success = false;
|
|
}
|
|
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/disableSharing', optionalJwt, async function(req, res) {
|
|
var type = req.body.type;
|
|
var uid = req.body.uid;
|
|
var is_playlist = req.body.is_playlist;
|
|
let success = null;
|
|
|
|
try {
|
|
success = true;
|
|
if (!is_playlist && type !== 'subscription') {
|
|
await db_api.updateRecord('files', {uid: uid}, {sharingEnabled: false})
|
|
} else if (is_playlist) {
|
|
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: false});
|
|
} else {
|
|
// error
|
|
success = false;
|
|
}
|
|
|
|
} catch(err) {
|
|
success = false;
|
|
}
|
|
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/incrementViewCount', async (req, res) => {
|
|
let file_uid = req.body.file_uid;
|
|
let sub_id = req.body.sub_id;
|
|
let uuid = req.body.uuid;
|
|
|
|
if (req.isAuthenticated()) uuid = req.user.uid;
|
|
|
|
const file_obj = await files_api.getVideo(file_uid, uuid, sub_id);
|
|
|
|
const current_view_count = file_obj && file_obj['local_view_count'] ? file_obj['local_view_count'] : 0;
|
|
const new_view_count = current_view_count + 1;
|
|
|
|
await db_api.setVideoProperty(file_uid, {local_view_count: new_view_count}, uuid, sub_id);
|
|
|
|
res.send({
|
|
success: true
|
|
});
|
|
});
|
|
|
|
// categories
|
|
|
|
app.post('/api/getAllCategories', optionalJwt, async (req, res) => {
|
|
const categories = await db_api.getRecords('categories');
|
|
res.send({categories: categories});
|
|
});
|
|
|
|
app.post('/api/createCategory', optionalJwt, async (req, res) => {
|
|
const name = req.body.name;
|
|
const new_category = {
|
|
name: name,
|
|
uid: uuid(),
|
|
rules: [],
|
|
custom_output: ''
|
|
};
|
|
|
|
await db_api.insertRecordIntoTable('categories', new_category);
|
|
|
|
res.send({
|
|
new_category: new_category,
|
|
success: !!new_category
|
|
});
|
|
});
|
|
|
|
app.post('/api/deleteCategory', optionalJwt, async (req, res) => {
|
|
const category_uid = req.body.category_uid;
|
|
|
|
await db_api.removeRecord('categories', {uid: category_uid});
|
|
|
|
res.send({
|
|
success: true
|
|
});
|
|
});
|
|
|
|
app.post('/api/updateCategory', optionalJwt, async (req, res) => {
|
|
const category = req.body.category;
|
|
await db_api.updateRecord('categories', {uid: category.uid}, category)
|
|
res.send({success: true});
|
|
});
|
|
|
|
app.post('/api/updateCategories', optionalJwt, async (req, res) => {
|
|
const categories = req.body.categories;
|
|
await db_api.removeAllRecords('categories');
|
|
await db_api.insertRecordsIntoTable('categories', categories);
|
|
res.send({success: true});
|
|
});
|
|
|
|
// subscriptions
|
|
|
|
app.post('/api/subscribe', optionalJwt, async (req, res) => {
|
|
let name = req.body.name;
|
|
let url = req.body.url;
|
|
let maxQuality = req.body.maxQuality;
|
|
let timerange = req.body.timerange;
|
|
let audioOnly = req.body.audioOnly;
|
|
let customArgs = req.body.customArgs;
|
|
let customOutput = req.body.customFileOutput;
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const new_sub = {
|
|
name: name,
|
|
url: url,
|
|
maxQuality: maxQuality,
|
|
id: uuid(),
|
|
user_uid: user_uid,
|
|
type: audioOnly ? 'audio' : 'video'
|
|
};
|
|
|
|
// adds timerange if it exists, otherwise all videos will be downloaded
|
|
if (timerange) {
|
|
new_sub.timerange = timerange;
|
|
}
|
|
|
|
if (customArgs && customArgs !== '') {
|
|
new_sub.custom_args = customArgs;
|
|
}
|
|
|
|
if (customOutput && customOutput !== '') {
|
|
new_sub.custom_output = customOutput;
|
|
}
|
|
|
|
const result_obj = await subscriptions_api.subscribe(new_sub, user_uid);
|
|
|
|
if (result_obj.success) {
|
|
res.send({
|
|
new_sub: new_sub
|
|
});
|
|
} else {
|
|
res.send({
|
|
new_sub: null,
|
|
error: result_obj.error
|
|
})
|
|
}
|
|
});
|
|
|
|
app.post('/api/unsubscribe', optionalJwt, async (req, res) => {
|
|
let deleteMode = req.body.deleteMode
|
|
let sub_id = req.body.sub_id;
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
let result_obj = await subscriptions_api.unsubscribe(sub_id, deleteMode, user_uid);
|
|
if (result_obj.success) {
|
|
res.send({
|
|
success: result_obj.success
|
|
});
|
|
} else {
|
|
res.send({
|
|
success: false,
|
|
error: result_obj.error
|
|
});
|
|
}
|
|
});
|
|
|
|
app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => {
|
|
let deleteForever = req.body.deleteForever;
|
|
let file_uid = req.body.file_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
let success = await files_api.deleteFile(file_uid, deleteForever, user_uid);
|
|
|
|
if (success) {
|
|
res.send({
|
|
success: success
|
|
});
|
|
} else {
|
|
res.sendStatus(500);
|
|
}
|
|
|
|
});
|
|
|
|
app.post('/api/getSubscription', optionalJwt, async (req, res) => {
|
|
let subID = req.body.id;
|
|
let subName = req.body.name; // if included, subID is optional
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
// get sub from db
|
|
let subscription = null;
|
|
if (subID) {
|
|
subscription = await subscriptions_api.getSubscription(subID, user_uid)
|
|
} else if (subName) {
|
|
subscription = await subscriptions_api.getSubscriptionByName(subName, user_uid)
|
|
}
|
|
|
|
if (!subscription) {
|
|
// failed to get subscription from db, send 400 error
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
subscription = JSON.parse(JSON.stringify(subscription));
|
|
|
|
// get sub videos
|
|
if (subscription.name) {
|
|
const sub_files_filter = {sub_id: subscription.id, ...getScopedFilterByUser(user_uid)};
|
|
var parsed_files = await db_api.getRecords('files', sub_files_filter); // subscription.videos;
|
|
subscription['videos'] = parsed_files;
|
|
// loop through files for extra processing
|
|
for (let i = 0; i < parsed_files.length; i++) {
|
|
const file = parsed_files[i];
|
|
// check if chat exists for twitch videos
|
|
if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
|
|
}
|
|
|
|
res.send({
|
|
subscription: subscription,
|
|
files: parsed_files
|
|
});
|
|
} else {
|
|
res.sendStatus(500);
|
|
}
|
|
});
|
|
|
|
app.post('/api/downloadVideosForSubscription', optionalJwt, async (req, res) => {
|
|
const subID = req.body.subID;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const sub = await subscriptions_api.getSubscription(subID, user_uid);
|
|
if (!sub) {
|
|
res.send({success: false});
|
|
return;
|
|
}
|
|
subscriptions_api.getVideosForSub(sub.id);
|
|
res.send({
|
|
success: true
|
|
});
|
|
});
|
|
|
|
app.post('/api/updateSubscription', optionalJwt, async (req, res) => {
|
|
const updated_sub = req.body.subscription;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await subscriptions_api.updateSubscription(updated_sub, user_uid);
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/checkSubscription', optionalJwt, async (req, res) => {
|
|
let sub_id = req.body.sub_id;
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await subscriptions_api.getVideosForSub(sub_id, user_uid);
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/cancelCheckSubscription', optionalJwt, async (req, res) => {
|
|
let sub_id = req.body.sub_id;
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await subscriptions_api.cancelCheckSubscription(sub_id, user_uid);
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/cancelSubscriptionCheck', optionalJwt, async (req, res) => {
|
|
let sub_id = req.body.sub_id;
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await subscriptions_api.getVideosForSub(sub_id, user_uid);
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
|
|
let user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
// get subs from api
|
|
let subscriptions = await subscriptions_api.getSubscriptions(user_uid);
|
|
|
|
res.send({
|
|
subscriptions: subscriptions
|
|
});
|
|
});
|
|
|
|
app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
|
|
let playlistName = req.body.playlistName;
|
|
let uids = req.body.uids;
|
|
|
|
const new_playlist = await files_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
|
|
|
|
res.send({
|
|
new_playlist: new_playlist,
|
|
success: !!new_playlist // always going to be true
|
|
})
|
|
});
|
|
|
|
app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
|
|
let playlist_id = req.body.playlist_id;
|
|
let uuid = req.body.uuid ? req.body.uuid : (req.user && req.user.uid ? req.user.uid : null);
|
|
let include_file_metadata = req.body.include_file_metadata;
|
|
if (req.user && req.user.uid) uuid = req.user.uid;
|
|
|
|
const playlist = await files_api.getPlaylist(playlist_id, uuid);
|
|
const file_objs = [];
|
|
|
|
if (playlist && include_file_metadata) {
|
|
for (let i = 0; i < playlist['uids'].length; i++) {
|
|
const uid = playlist['uids'][i];
|
|
const file_obj = await files_api.getVideo(uid, uuid);
|
|
if (file_obj) file_objs.push(file_obj);
|
|
// TODO: remove file from playlist if could not be found
|
|
}
|
|
}
|
|
|
|
res.send({
|
|
playlist: playlist,
|
|
file_objs: file_objs,
|
|
success: !!playlist
|
|
});
|
|
});
|
|
|
|
app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
const include_categories = req.body.include_categories;
|
|
const filter_obj = getScopedFilterByUser(uuid);
|
|
|
|
let playlists = await db_api.getRecords('playlists', filter_obj);
|
|
if (include_categories) {
|
|
const categories = await categories_api.getCategoriesAsPlaylists();
|
|
if (categories) {
|
|
playlists = playlists.concat(categories);
|
|
}
|
|
}
|
|
|
|
res.send({
|
|
playlists: playlists
|
|
});
|
|
});
|
|
|
|
app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => {
|
|
let playlist_id = req.body.playlist_id;
|
|
let file_uid = req.body.file_uid;
|
|
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const playlist = await files_api.getPlaylist(playlist_id, user_uid);
|
|
if (!playlist) {
|
|
res.send({
|
|
success: false
|
|
});
|
|
return;
|
|
}
|
|
const file_obj = await files_api.getVideo(file_uid, user_uid);
|
|
if (!file_obj) {
|
|
res.send({
|
|
success: false
|
|
});
|
|
return;
|
|
}
|
|
|
|
playlist.uids.push(file_uid);
|
|
|
|
let success = await files_api.updatePlaylist(playlist, user_uid);
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/updatePlaylist', optionalJwt, async (req, res) => {
|
|
let playlist = req.body.playlist;
|
|
let success = await files_api.updatePlaylist(playlist, req.user && req.user.uid);
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/deletePlaylist', optionalJwt, async (req, res) => {
|
|
let playlistID = req.body.playlist_id;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
let success = null;
|
|
try {
|
|
// removes playlist from playlists
|
|
const filter_obj = {id: playlistID, ...getScopedFilterByUser(user_uid)};
|
|
await db_api.removeRecord('playlists', filter_obj)
|
|
|
|
success = true;
|
|
} catch(e) {
|
|
success = false;
|
|
}
|
|
|
|
res.send({
|
|
success: success
|
|
})
|
|
});
|
|
|
|
// deletes non-subscription files
|
|
app.post('/api/deleteFile', optionalJwt, async (req, res) => {
|
|
const uid = req.body.uid;
|
|
const blacklistMode = req.body.blacklistMode;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
let wasDeleted = false;
|
|
wasDeleted = await files_api.deleteFile(uid, blacklistMode, user_uid);
|
|
res.send(wasDeleted);
|
|
});
|
|
|
|
app.post('/api/deleteAllFiles', optionalJwt, async (req, res) => {
|
|
const blacklistMode = false;
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
let files = null;
|
|
let text_search = req.body.text_search;
|
|
let file_type_filter = req.body.file_type_filter;
|
|
|
|
const filter_obj = getScopedFilterByUser(uuid);
|
|
const regex = true;
|
|
if (text_search) {
|
|
if (regex) {
|
|
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
|
|
} else {
|
|
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
|
|
}
|
|
}
|
|
|
|
if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
|
|
else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
|
|
|
|
files = await db_api.getRecords('files', filter_obj);
|
|
|
|
let file_count = await db_api.getRecords('files', filter_obj, true);
|
|
let delete_count = 0;
|
|
|
|
for (let i = 0; i < files.length; i++) {
|
|
let wasDeleted = false;
|
|
wasDeleted = await files_api.deleteFile(files[i].uid, blacklistMode, uuid);
|
|
if (wasDeleted) {
|
|
delete_count++;
|
|
}
|
|
}
|
|
|
|
res.send({
|
|
file_count: file_count,
|
|
delete_count: delete_count
|
|
});
|
|
});
|
|
|
|
app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
|
|
let uid = req.body.uid;
|
|
let uuid = req.body.uuid;
|
|
let playlist_id = req.body.playlist_id;
|
|
let sub_id = req.body.sub_id;
|
|
|
|
let file_path_to_download = null;
|
|
|
|
if (req.user && req.user.uid) uuid = req.user.uid;
|
|
|
|
let zip_file_generated = false;
|
|
if (playlist_id) {
|
|
zip_file_generated = true;
|
|
const playlist_files_to_download = [];
|
|
const playlist = await files_api.getPlaylist(playlist_id, uuid);
|
|
if (!playlist) {
|
|
res.sendStatus(404);
|
|
return;
|
|
}
|
|
for (let i = 0; i < playlist['uids'].length; i++) {
|
|
const playlist_file_uid = playlist['uids'][i];
|
|
const file_obj = await files_api.getVideo(playlist_file_uid, uuid);
|
|
if (file_obj) playlist_files_to_download.push(file_obj);
|
|
}
|
|
|
|
// generate zip
|
|
file_path_to_download = await utils.createContainerZipFile(playlist['name'], playlist_files_to_download);
|
|
} else if (sub_id && !uid) {
|
|
zip_file_generated = true;
|
|
const sub = await subscriptions_api.getSubscription(sub_id, req.isAuthenticated() ? req.user.uid : null);
|
|
if (!sub) {
|
|
res.sendStatus(404);
|
|
return;
|
|
}
|
|
const sub_files_to_download = await db_api.getRecords('files', {sub_id: sub_id, ...getScopedFilterByUser(req.isAuthenticated() ? req.user.uid : null)});
|
|
|
|
// generate zip
|
|
file_path_to_download = await utils.createContainerZipFile(sub['name'], sub_files_to_download);
|
|
} else {
|
|
const file_obj = await files_api.getVideo(uid, uuid, sub_id)
|
|
if (!file_obj) {
|
|
res.sendStatus(404);
|
|
return;
|
|
}
|
|
file_path_to_download = file_obj.path;
|
|
}
|
|
if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
|
|
res.sendFile(file_path_to_download, function (err) {
|
|
if (err) {
|
|
logger.error(err);
|
|
} else if (zip_file_generated) {
|
|
try {
|
|
// delete generated zip file
|
|
fs.unlinkSync(file_path_to_download);
|
|
} catch(e) {
|
|
logger.error(`Failed to remove file after sending to client: ${file_path_to_download}`);
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
app.post('/api/getArchives', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
const sub_id = req.body.sub_id;
|
|
const filter_obj = {sub_id: sub_id, ...getScopedFilterByUser(uuid)};
|
|
const type = req.body.type;
|
|
|
|
// we do this for file types because if type is null, that means get files of all types
|
|
if (type) filter_obj['type'] = type;
|
|
|
|
const archives = await db_api.getRecords('archives', filter_obj);
|
|
|
|
res.send({
|
|
archives: archives
|
|
});
|
|
});
|
|
|
|
app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
const sub_id = req.body.sub_id;
|
|
const type = req.body.type;
|
|
|
|
const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
|
|
|
|
if (archive_text !== null && archive_text !== undefined) {
|
|
res.setHeader('Content-type', "application/octet-stream");
|
|
res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
|
|
res.send(archive_text);
|
|
} else {
|
|
res.sendStatus(400);
|
|
}
|
|
|
|
});
|
|
|
|
app.post('/api/importArchive', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
const archive = req.body.archive;
|
|
const sub_id = req.body.sub_id;
|
|
const type = req.body.type;
|
|
|
|
const archive_text = Buffer.from(archive.split(',')[1], 'base64').toString();
|
|
|
|
const imported_count = await archive_api.importArchiveFile(archive_text, type, uuid, sub_id);
|
|
|
|
res.send({
|
|
success: !!imported_count,
|
|
imported_count: imported_count
|
|
});
|
|
});
|
|
|
|
app.post('/api/deleteArchiveItems', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
const archives = req.body.archives;
|
|
|
|
let success = true;
|
|
for (const archive of archives) {
|
|
success &= await archive_api.removeFromArchive(archive['extractor'], archive['id'], archive['type'], uuid, archive['sub_id']);
|
|
}
|
|
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
var upload_multer = multer({ dest: __dirname + '/appdata/' });
|
|
app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) => {
|
|
if (!req.file || !req.file.path) {
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
const new_path = path.join(__dirname, 'appdata', 'cookies.txt');
|
|
const uploadBasePath = path.join(__dirname, 'appdata');
|
|
const resolvedUploadedPath = path.resolve(req.file.path);
|
|
const relativeUploadedPath = path.relative(uploadBasePath, resolvedUploadedPath);
|
|
if (relativeUploadedPath.startsWith('..') || path.isAbsolute(relativeUploadedPath)) {
|
|
logger.error(`Refusing to move uploaded cookies file outside appdata: ${req.file.path}`);
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
if (await fs.pathExists(resolvedUploadedPath)) {
|
|
await fs.rename(resolvedUploadedPath, new_path);
|
|
} else {
|
|
res.sendStatus(500);
|
|
return;
|
|
}
|
|
|
|
if (await fs.pathExists(new_path)) {
|
|
res.send({success: true});
|
|
} else {
|
|
res.sendStatus(500);
|
|
}
|
|
|
|
});
|
|
|
|
function getCookiesFileSummary(cookies_text) {
|
|
const lines = cookies_text.split(/\r?\n/).map(line => line.trim()).filter(line => line.length > 0);
|
|
const cookie_lines = lines.filter(line => !line.startsWith('#') || line.startsWith('#HttpOnly_'));
|
|
let invalid_entries = 0;
|
|
|
|
for (const line of cookie_lines) {
|
|
// Netscape cookie format should contain at least 7 tab-separated values.
|
|
if (line.split('\t').length < 7) invalid_entries++;
|
|
}
|
|
|
|
return {
|
|
total_lines: lines.length,
|
|
cookie_entries: cookie_lines.length,
|
|
invalid_entries: invalid_entries
|
|
};
|
|
}
|
|
|
|
function normalizeCookieTestError(err) {
|
|
if (!err) return 'Unknown error.';
|
|
|
|
let message = null;
|
|
if (typeof err === 'string') {
|
|
message = err;
|
|
} else if (err.stderr) {
|
|
message = err.stderr.toString();
|
|
} else if (err.message) {
|
|
message = err.message.toString();
|
|
} else {
|
|
message = JSON.stringify(err);
|
|
}
|
|
|
|
if (!message) return 'Unknown error.';
|
|
const max_error_length = 1200;
|
|
return message.length > max_error_length ? message.substring(0, max_error_length) + '...' : message;
|
|
}
|
|
|
|
app.post('/api/testCookies', testCookiesRateLimiter, optionalJwt, async (req, res) => {
|
|
const logs = [];
|
|
const use_cookies_enabled = config_api.getConfigItem('ytdl_use_cookies');
|
|
const downloader = config_api.getConfigItem('ytdl_default_downloader');
|
|
const test_url = req.body && req.body.url ? req.body.url.trim() : '';
|
|
const cookie_path = path.join(__dirname, 'appdata', 'cookies.txt');
|
|
const relative_cookie_path = path.join('appdata', 'cookies.txt');
|
|
|
|
logs.push('Starting cookie test.');
|
|
logs.push(`Downloader: ${downloader}`);
|
|
logs.push(`Use Cookies setting is ${use_cookies_enabled ? 'enabled' : 'disabled'}.`);
|
|
|
|
if (!test_url) {
|
|
logs.push('No URL was provided for cookie testing.');
|
|
res.status(400).send({
|
|
success: false,
|
|
error: 'Missing URL to test.',
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: false
|
|
});
|
|
return;
|
|
}
|
|
|
|
let parsed_test_url = null;
|
|
try {
|
|
parsed_test_url = new URL(test_url);
|
|
} catch (err) {
|
|
parsed_test_url = null;
|
|
}
|
|
|
|
if (!parsed_test_url || (parsed_test_url.protocol !== 'http:' && parsed_test_url.protocol !== 'https:')) {
|
|
logs.push(`Invalid test URL provided: ${test_url}`);
|
|
res.status(400).send({
|
|
success: false,
|
|
error: 'Invalid URL. Only http/https URLs are allowed.',
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: false
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (!(await fs.pathExists(cookie_path))) {
|
|
logs.push(`Cookie file was not found at ${cookie_path}.`);
|
|
res.send({
|
|
success: false,
|
|
error: 'Cookies file not found.',
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: false
|
|
});
|
|
return;
|
|
}
|
|
|
|
const cookie_stats = await fs.stat(cookie_path);
|
|
logs.push(`Cookie file found (${cookie_stats.size} bytes).`);
|
|
|
|
if (cookie_stats.size === 0) {
|
|
logs.push('Cookie file is empty.');
|
|
res.send({
|
|
success: false,
|
|
error: 'Cookies file is empty.',
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: true,
|
|
cookie_file_size: cookie_stats.size
|
|
});
|
|
return;
|
|
}
|
|
|
|
const cookies_text = await fs.readFile(cookie_path, 'utf8');
|
|
const cookie_summary = getCookiesFileSummary(cookies_text);
|
|
|
|
logs.push(`Detected ${cookie_summary.cookie_entries} cookie entries from ${cookie_summary.total_lines} non-empty lines.`);
|
|
if (cookie_summary.invalid_entries > 0) {
|
|
logs.push(`Detected ${cookie_summary.invalid_entries} entries that may not be valid Netscape cookie rows.`);
|
|
}
|
|
|
|
const args = [
|
|
'--skip-download',
|
|
'--no-warnings',
|
|
'--no-playlist',
|
|
'--dump-single-json',
|
|
'--cookies',
|
|
relative_cookie_path
|
|
];
|
|
logs.push(`Testing URL: ${test_url}`);
|
|
logs.push(`Executing test command with cookies at ${relative_cookie_path}.`);
|
|
|
|
let run_response = null;
|
|
try {
|
|
run_response = await youtubedl_api.runYoutubeDL(test_url, args);
|
|
} catch (err) {
|
|
const error_message = normalizeCookieTestError(err);
|
|
logs.push(`Failed to start downloader process. ${error_message}`);
|
|
res.status(500).send({
|
|
success: false,
|
|
error: error_message,
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: true,
|
|
cookie_summary: cookie_summary
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (!run_response || !run_response.callback) {
|
|
logs.push('Downloader process did not initialize correctly.');
|
|
res.status(500).send({
|
|
success: false,
|
|
error: 'Failed to initialize downloader process.',
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: true,
|
|
cookie_summary: cookie_summary
|
|
});
|
|
return;
|
|
}
|
|
|
|
const {parsed_output, err} = await run_response.callback;
|
|
if (parsed_output && parsed_output.length > 0) {
|
|
const info_obj = parsed_output[0];
|
|
const title = info_obj && info_obj.title ? info_obj.title : null;
|
|
const extractor = info_obj && info_obj.extractor ? info_obj.extractor : null;
|
|
|
|
if (title) logs.push(`Metadata fetch succeeded: "${title}".`);
|
|
else logs.push('Metadata fetch succeeded.');
|
|
|
|
if (extractor) logs.push(`Extractor used: ${extractor}.`);
|
|
|
|
res.send({
|
|
success: true,
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: true,
|
|
cookie_file_size: cookie_stats.size,
|
|
cookie_summary: cookie_summary,
|
|
result: {
|
|
title: title,
|
|
extractor: extractor
|
|
}
|
|
});
|
|
return;
|
|
}
|
|
|
|
const error_message = normalizeCookieTestError(err);
|
|
logs.push('Metadata fetch failed while using cookies.');
|
|
logs.push(error_message);
|
|
res.send({
|
|
success: false,
|
|
error: error_message,
|
|
logs: logs,
|
|
use_cookies_enabled: use_cookies_enabled,
|
|
cookie_file_found: true,
|
|
cookie_file_size: cookie_stats.size,
|
|
cookie_summary: cookie_summary
|
|
});
|
|
});
|
|
|
|
// Updater API calls
|
|
|
|
app.get('/api/updaterStatus', optionalJwt, async (req, res) => {
|
|
let status = updaterStatus;
|
|
|
|
if (status) {
|
|
res.send(updaterStatus);
|
|
} else {
|
|
res.sendStatus(404);
|
|
}
|
|
|
|
});
|
|
|
|
app.post('/api/updateServer', optionalJwt, async (req, res) => {
|
|
let tag = req.body.tag;
|
|
|
|
updateServer(tag);
|
|
|
|
res.send({
|
|
success: true
|
|
});
|
|
|
|
});
|
|
|
|
// API Key API calls
|
|
|
|
app.post('/api/generateNewAPIKey', optionalJwt, function (req, res) {
|
|
const new_api_key = uuid();
|
|
config_api.setConfigItem('ytdl_api_key', new_api_key);
|
|
res.send({new_api_key: new_api_key});
|
|
});
|
|
|
|
// Streaming API calls
|
|
|
|
app.get('/api/stream', optionalJwt, async (req, res) => {
|
|
const type = req.query.type;
|
|
const uuid = req.user ? req.user.uid : (req.query.uuid ? req.query.uuid : null);
|
|
const sub_id = req.query.sub_id;
|
|
const mimetype = type === 'audio' ? 'audio/mp3' : 'video/mp4';
|
|
var head;
|
|
let uid = decodeURIComponent(req.query.uid);
|
|
|
|
let file_path = null;
|
|
let file_obj = null;
|
|
|
|
const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
|
|
if (!multiUserMode || req.isAuthenticated() || req.can_watch) {
|
|
file_obj = await files_api.getVideo(uid, uuid, sub_id);
|
|
if (file_obj) file_path = file_obj['path'];
|
|
else file_path = null;
|
|
}
|
|
if (!fs.existsSync(file_path)) {
|
|
logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
|
|
return;
|
|
}
|
|
const stat = fs.statSync(file_path);
|
|
const fileSize = stat.size;
|
|
const range = req.headers.range;
|
|
if (range) {
|
|
const parts = range.replace(/bytes=/, "").split("-")
|
|
const start = parseInt(parts[0], 10)
|
|
const end = parts[1]
|
|
? parseInt(parts[1], 10)
|
|
: fileSize-1
|
|
const chunksize = (end-start)+1
|
|
const file = fs.createReadStream(file_path, {start, end})
|
|
if (config_api.descriptors[uid]) config_api.descriptors[uid].push(file);
|
|
else config_api.descriptors[uid] = [file];
|
|
file.on('close', function() {
|
|
let index = config_api.descriptors[uid].indexOf(file);
|
|
config_api.descriptors[uid].splice(index, 1);
|
|
logger.debug('Successfully closed stream and removed file reference.');
|
|
});
|
|
head = {
|
|
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
|
|
'Accept-Ranges': 'bytes',
|
|
'Content-Length': chunksize,
|
|
'Content-Type': mimetype,
|
|
}
|
|
res.writeHead(206, head);
|
|
file.pipe(res);
|
|
} else {
|
|
head = {
|
|
'Content-Length': fileSize,
|
|
'Content-Type': mimetype,
|
|
}
|
|
res.writeHead(200, head)
|
|
fs.createReadStream(file_path).pipe(res)
|
|
}
|
|
});
|
|
|
|
app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
|
|
// Express route params are already decoded.
|
|
const requestedPath = typeof req.params.path === 'string' ? req.params.path : '';
|
|
const resolvedRequestedPath = path.isAbsolute(requestedPath) ? path.resolve(requestedPath) : path.resolve(__dirname, requestedPath);
|
|
const thumbnailExt = path.extname(resolvedRequestedPath).toLowerCase();
|
|
const allowedThumbnailExts = new Set(['.jpg', '.jpeg', '.png', '.webp']);
|
|
if (!allowedThumbnailExts.has(thumbnailExt)) {
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
const configuredRoots = [
|
|
__dirname,
|
|
config_api.getConfigItem('ytdl_audio_folder_path'),
|
|
config_api.getConfigItem('ytdl_video_folder_path'),
|
|
config_api.getConfigItem('ytdl_subscriptions_base_path'),
|
|
config_api.getConfigItem('ytdl_users_base_path')
|
|
]
|
|
.filter(Boolean)
|
|
.map(root => path.resolve(__dirname, root));
|
|
|
|
let thumbnailRoot = null;
|
|
let thumbnailRelativePath = null;
|
|
for (const root of configuredRoots) {
|
|
const relativePath = path.relative(root, resolvedRequestedPath);
|
|
if (relativePath && !relativePath.startsWith('..') && !path.isAbsolute(relativePath)) {
|
|
thumbnailRoot = root;
|
|
thumbnailRelativePath = relativePath;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!thumbnailRoot || !thumbnailRelativePath) {
|
|
logger.warn(`Refusing thumbnail request outside allowed roots: ${requestedPath}`);
|
|
res.sendStatus(403);
|
|
return;
|
|
}
|
|
|
|
res.sendFile(thumbnailRelativePath, { root: thumbnailRoot }, (err) => {
|
|
if (!err) return;
|
|
if (res.headersSent) return;
|
|
if (err.statusCode === 404) {
|
|
res.sendStatus(404);
|
|
return;
|
|
}
|
|
logger.error(err);
|
|
res.sendStatus(500);
|
|
});
|
|
});
|
|
|
|
// Downloads management
|
|
|
|
app.post('/api/downloads', optionalJwt, async (req, res) => {
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const uids = req.body.uids;
|
|
const filter_obj = getScopedFilterByUser(user_uid);
|
|
let downloads = await db_api.getRecords('download_queue', filter_obj);
|
|
|
|
if (uids) downloads = downloads.filter(download => uids.includes(download['uid']));
|
|
|
|
res.send({downloads: downloads});
|
|
});
|
|
|
|
app.post('/api/download', optionalJwt, async (req, res) => {
|
|
const download_uid = req.body.download_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const filter_obj = {uid: download_uid, ...getScopedFilterByUser(user_uid)};
|
|
|
|
const download = await db_api.getRecord('download_queue', filter_obj);
|
|
|
|
if (download) {
|
|
res.send({download: download});
|
|
} else {
|
|
res.send({download: null});
|
|
}
|
|
});
|
|
|
|
app.post('/api/clearDownloads', optionalJwt, async (req, res) => {
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const scoped_filter = getScopedFilterByUser(user_uid);
|
|
const clear_finished = req.body.clear_finished;
|
|
const clear_paused = req.body.clear_paused;
|
|
const clear_errors = req.body.clear_errors;
|
|
let success = true;
|
|
if (clear_finished) success &= await db_api.removeAllRecords('download_queue', {finished: true, ...scoped_filter, error: null});
|
|
if (clear_paused) success &= await db_api.removeAllRecords('download_queue', {paused: true, ...scoped_filter});
|
|
if (clear_errors) success &= await db_api.removeAllRecords('download_queue', {error: {$ne: null}, ...scoped_filter});
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/clearDownload', optionalJwt, async (req, res) => {
|
|
const download_uid = req.body.download_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const owned_download = await db_api.getRecord('download_queue', {uid: download_uid, ...getScopedFilterByUser(user_uid)});
|
|
if (!owned_download) {
|
|
res.send({success: false});
|
|
return;
|
|
}
|
|
const success = await downloader_api.clearDownload(download_uid);
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/pauseDownload', optionalJwt, async (req, res) => {
|
|
const download_uid = req.body.download_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const owned_download = await db_api.getRecord('download_queue', {uid: download_uid, ...getScopedFilterByUser(user_uid)});
|
|
if (!owned_download) {
|
|
res.send({success: false});
|
|
return;
|
|
}
|
|
const success = await downloader_api.pauseDownload(download_uid);
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/pauseAllDownloads', optionalJwt, async (req, res) => {
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
let success = true;
|
|
const all_running_downloads = await db_api.getRecords('download_queue', {paused: false, finished: false, ...getScopedFilterByUser(user_uid)});
|
|
for (let i = 0; i < all_running_downloads.length; i++) {
|
|
success &= await downloader_api.pauseDownload(all_running_downloads[i]['uid']);
|
|
}
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/resumeDownload', optionalJwt, async (req, res) => {
|
|
const download_uid = req.body.download_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const owned_download = await db_api.getRecord('download_queue', {uid: download_uid, ...getScopedFilterByUser(user_uid)});
|
|
if (!owned_download) {
|
|
res.send({success: false});
|
|
return;
|
|
}
|
|
const success = await downloader_api.resumeDownload(download_uid);
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/resumeAllDownloads', optionalJwt, async (req, res) => {
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
let success = true;
|
|
const all_paused_downloads = await db_api.getRecords('download_queue', {paused: true, ...getScopedFilterByUser(user_uid), error: null});
|
|
for (let i = 0; i < all_paused_downloads.length; i++) {
|
|
success &= await downloader_api.resumeDownload(all_paused_downloads[i]['uid']);
|
|
}
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/restartDownload', optionalJwt, async (req, res) => {
|
|
const download_uid = req.body.download_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const owned_download = await db_api.getRecord('download_queue', {uid: download_uid, ...getScopedFilterByUser(user_uid)});
|
|
if (!owned_download) {
|
|
res.send({success: false, new_download_uid: null});
|
|
return;
|
|
}
|
|
const new_download = await downloader_api.restartDownload(download_uid);
|
|
res.send({success: !!new_download, new_download_uid: new_download ? new_download['uid'] : null});
|
|
});
|
|
|
|
app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
|
|
const download_uid = req.body.download_uid;
|
|
const user_uid = req.isAuthenticated() ? req.user.uid : null;
|
|
const owned_download = await db_api.getRecord('download_queue', {uid: download_uid, ...getScopedFilterByUser(user_uid)});
|
|
if (!owned_download) {
|
|
res.send({success: false});
|
|
return;
|
|
}
|
|
const success = await downloader_api.cancelDownload(download_uid);
|
|
res.send({success: success});
|
|
});
|
|
|
|
// tasks
|
|
|
|
app.post('/api/getTasks', optionalJwt, async (req, res) => {
|
|
const tasks = await db_api.getRecords('tasks');
|
|
for (let task of tasks) {
|
|
if (!tasks_api.TASKS[task['key']]) {
|
|
logger.verbose(`Task ${task['key']} does not exist!`);
|
|
continue;
|
|
}
|
|
if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task['key']]['job'].nextInvocation().getTime();
|
|
}
|
|
res.send({tasks: tasks});
|
|
});
|
|
|
|
app.post('/api/resetTasks', optionalJwt, async (req, res) => {
|
|
const tasks_keys = Object.keys(tasks_api.TASKS);
|
|
for (let i = 0; i < tasks_keys.length; i++) {
|
|
const task_key = tasks_keys[i];
|
|
tasks_api.TASKS[task_key]['job'] = null;
|
|
}
|
|
await db_api.removeAllRecords('tasks');
|
|
await tasks_api.setupTasks();
|
|
res.send({success: true});
|
|
});
|
|
|
|
app.post('/api/getTask', optionalJwt, async (req, res) => {
|
|
const task_key = req.body.task_key;
|
|
const task = await db_api.getRecord('tasks', {key: task_key});
|
|
if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task_key]['job'].nextInvocation().getTime();
|
|
res.send({task: task});
|
|
});
|
|
|
|
app.post('/api/runTask', optionalJwt, async (req, res) => {
|
|
const task_key = req.body.task_key;
|
|
const task = await db_api.getRecord('tasks', {key: task_key});
|
|
|
|
let success = true;
|
|
if (task['running'] || task['confirming']) success = false;
|
|
else await tasks_api.executeRun(task_key);
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/confirmTask', optionalJwt, async (req, res) => {
|
|
const task_key = req.body.task_key;
|
|
const task = await db_api.getRecord('tasks', {key: task_key});
|
|
|
|
let success = true;
|
|
if (task['running'] || task['confirming'] || !task['data']) success = false;
|
|
else await tasks_api.executeConfirm(task_key);
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/updateTaskSchedule', optionalJwt, async (req, res) => {
|
|
const task_key = req.body.task_key;
|
|
const new_schedule = req.body.new_schedule;
|
|
|
|
await tasks_api.updateTaskSchedule(task_key, new_schedule);
|
|
|
|
res.send({success: true});
|
|
});
|
|
|
|
app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
|
|
const task_key = req.body.task_key;
|
|
const new_data = req.body.new_data;
|
|
|
|
const success = await db_api.updateRecord('tasks', {key: task_key}, {data: new_data});
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/updateTaskOptions', optionalJwt, async (req, res) => {
|
|
const task_key = req.body.task_key;
|
|
const new_options = req.body.new_options;
|
|
|
|
const success = await db_api.updateRecord('tasks', {key: task_key}, {options: new_options});
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
|
|
const backup_dir = path.join('appdata', 'db_backup');
|
|
fs.ensureDirSync(backup_dir);
|
|
const db_backups = [];
|
|
|
|
const candidate_backups = await utils.recFindByExt(backup_dir, 'bak', null, [], false);
|
|
for (let i = 0; i < candidate_backups.length; i++) {
|
|
const candidate_backup = candidate_backups[i];
|
|
|
|
// must have specific format
|
|
if (candidate_backup.split('.').length - 1 !== 4) continue;
|
|
|
|
const candidate_backup_path = candidate_backup;
|
|
const stats = fs.statSync(candidate_backup_path);
|
|
|
|
db_backups.push({ name: path.basename(candidate_backup), timestamp: parseInt(candidate_backup.split('.')[2]), size: stats.size, source: candidate_backup.includes('local') ? 'local' : 'remote' });
|
|
}
|
|
|
|
db_backups.sort((a,b) => b.timestamp - a.timestamp);
|
|
|
|
res.send({db_backups: db_backups});
|
|
});
|
|
|
|
app.post('/api/restoreDBBackup', optionalJwt, async (req, res) => {
|
|
const file_name = req.body.file_name;
|
|
|
|
const success = await db_api.restoreDB(file_name);
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
// logs management
|
|
|
|
app.post('/api/logs', optionalJwt, async function(req, res) {
|
|
let logs = null;
|
|
let lines = req.body.lines;
|
|
const logs_path = path.join('appdata', 'logs', 'combined.log')
|
|
if (await fs.pathExists(logs_path)) {
|
|
if (lines) logs = await read_last_lines.read(logs_path, lines);
|
|
else logs = await fs.readFile(logs_path, 'utf8');
|
|
}
|
|
else
|
|
logger.error(`Failed to find logs file at the expected location: ${logs_path}`)
|
|
|
|
res.send({
|
|
logs: logs,
|
|
success: !!logs
|
|
});
|
|
});
|
|
|
|
app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
|
|
const logs_path = path.join('appdata', 'logs', 'combined.log');
|
|
const logs_err_path = path.join('appdata', 'logs', 'error.log');
|
|
let success = false;
|
|
try {
|
|
await Promise.all([
|
|
fs.writeFile(logs_path, ''),
|
|
fs.writeFile(logs_err_path, '')
|
|
])
|
|
success = true;
|
|
} catch(e) {
|
|
logger.error(e);
|
|
}
|
|
|
|
res.send({
|
|
success: success
|
|
});
|
|
});
|
|
|
|
app.post('/api/getFileFormats', optionalJwt, async (req, res) => {
|
|
const url = req.body.url;
|
|
const result = await downloader_api.getVideoInfoByURL(url);
|
|
res.send({
|
|
result: result && result.length === 1 ? result[0] : null,
|
|
success: result && result.length === 0
|
|
})
|
|
});
|
|
|
|
// user authentication
|
|
|
|
app.get('/api/auth/oidc/status', (req, res) => {
|
|
res.send(oidc_api.getStatus());
|
|
});
|
|
|
|
app.get('/api/auth/oidc/login', async (req, res) => {
|
|
if (!oidc_api.isEnabled()) {
|
|
res.status(404).send('OIDC is disabled.');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const return_to = req.query.returnTo ? String(req.query.returnTo) : '/home';
|
|
const authorization_url = await oidc_api.createAuthorizationURL(return_to);
|
|
res.redirect(authorization_url);
|
|
} catch (err) {
|
|
logger.error(`OIDC login redirect failed: ${err.message}`);
|
|
res.sendStatus(500);
|
|
}
|
|
});
|
|
|
|
app.get('/api/auth/oidc/callback', async (req, res) => {
|
|
if (!oidc_api.isEnabled()) {
|
|
res.status(404).send('OIDC is disabled.');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const callback_result = await oidc_api.consumeAuthorizationCallback(req);
|
|
const claims = callback_result.claims || {};
|
|
if (!oidc_api.isClaimsAllowed(claims)) {
|
|
logger.error('OIDC login rejected: user does not match allowed groups policy.');
|
|
res.sendStatus(403);
|
|
return;
|
|
}
|
|
|
|
const oidc_config = oidc_api.getConfiguration();
|
|
const user_obj = await auth_api.upsertOIDCUser(claims, {
|
|
auto_register: oidc_config.auto_register,
|
|
admin_claim: oidc_config.admin_claim,
|
|
admin_value: oidc_config.admin_value,
|
|
groups_claim: oidc_config.groups_claim,
|
|
username_claim: oidc_config.username_claim,
|
|
display_name_claim: oidc_config.display_name_claim
|
|
});
|
|
if (!user_obj) {
|
|
res.sendStatus(403);
|
|
return;
|
|
}
|
|
|
|
const auth_response = await auth_api.getAuthResponseObject(user_obj);
|
|
const return_to = callback_result.return_to ? callback_result.return_to : '/home';
|
|
const redirect_path = `${getOrigin()}/#/login;oidc_token=${encodeURIComponent(auth_response.token)};redirect=${encodeURIComponent(return_to)}`;
|
|
res.setHeader('Set-Cookie', 'ytdl_oidc_bootstrap=1; Path=/; Max-Age=60; HttpOnly; SameSite=Lax');
|
|
res.redirect(redirect_path);
|
|
} catch (err) {
|
|
logger.error(`OIDC callback failed: ${err.message}`);
|
|
res.sendStatus(401);
|
|
}
|
|
});
|
|
|
|
app.post('/api/auth/register', optionalJwt, async (req, res) => {
|
|
if (oidc_api.isEnabled()) {
|
|
res.status(403).send('Registration is disabled when OIDC is enabled.');
|
|
return;
|
|
}
|
|
|
|
const userid = req.body.userid;
|
|
const username = req.body.username;
|
|
const plaintextPassword = req.body.password;
|
|
|
|
if (userid !== 'admin' && !config_api.getConfigItem('ytdl_allow_registration') && !req.isAuthenticated() && (!req.user || !exports.userHasPermission(req.user.uid, 'settings'))) {
|
|
logger.error(`Registration failed for user ${userid}. Registration is disabled.`);
|
|
res.sendStatus(409);
|
|
return;
|
|
}
|
|
|
|
if (plaintextPassword === "") {
|
|
logger.error(`Registration failed for user ${userid}. A password must be provided.`);
|
|
res.sendStatus(409);
|
|
return;
|
|
}
|
|
|
|
if (!userid || !username) {
|
|
logger.error(`Registration failed for user ${userid}. Username or userid is invalid.`);
|
|
}
|
|
|
|
const new_user = await auth_api.registerUser(userid, username, plaintextPassword);
|
|
|
|
if (!new_user) {
|
|
res.sendStatus(409);
|
|
return;
|
|
}
|
|
|
|
res.send({
|
|
user: new_user
|
|
});
|
|
});
|
|
app.post('/api/auth/login'
|
|
, (req, res, next) => {
|
|
if (oidc_api.isEnabled()) {
|
|
res.status(403).send('Password login is disabled when OIDC is enabled.');
|
|
return;
|
|
}
|
|
next();
|
|
}
|
|
, auth_api.passport.authenticate(['local', 'ldapauth'], { session: false })
|
|
, auth_api.generateJWT
|
|
, auth_api.returnAuthResponse
|
|
);
|
|
app.post('/api/auth/jwtAuth'
|
|
, auth_api.passport.authenticate('jwt', { session: false })
|
|
, auth_api.passport.authorize('jwt')
|
|
, auth_api.generateJWT
|
|
, auth_api.returnAuthResponse
|
|
);
|
|
app.post('/api/auth/changePassword', optionalJwt, async (req, res) => {
|
|
let user_uid = req.body.user_uid;
|
|
let password = req.body.new_password;
|
|
let success = await auth_api.changeUserPassword(user_uid, password);
|
|
res.send({success: success});
|
|
});
|
|
app.post('/api/auth/adminExists', async (req, res) => {
|
|
let exists = await auth_api.adminExists();
|
|
res.send({exists: exists});
|
|
});
|
|
|
|
// user management
|
|
app.post('/api/getUsers', optionalJwt, async (req, res) => {
|
|
let users = await db_api.getRecords('users');
|
|
res.send({users: users});
|
|
});
|
|
app.post('/api/getRoles', optionalJwt, async (req, res) => {
|
|
let roles = await db_api.getRecords('roles');
|
|
res.send({roles: roles});
|
|
});
|
|
|
|
app.post('/api/updateUser', optionalJwt, async (req, res) => {
|
|
let change_obj = req.body.change_object;
|
|
try {
|
|
if (change_obj.name) {
|
|
await db_api.updateRecord('users', {uid: change_obj.uid}, {name: change_obj.name});
|
|
}
|
|
if (change_obj.role) {
|
|
await db_api.updateRecord('users', {uid: change_obj.uid}, {role: change_obj.role});
|
|
}
|
|
res.send({success: true});
|
|
} catch (err) {
|
|
logger.error(err);
|
|
res.send({success: false});
|
|
}
|
|
});
|
|
|
|
app.post('/api/deleteUser', optionalJwt, async (req, res) => {
|
|
let uid = req.body.uid;
|
|
try {
|
|
const success = await auth_api.deleteUser(uid);
|
|
res.send({success: success});
|
|
} catch (err) {
|
|
logger.error(err);
|
|
res.send({success: false});
|
|
}
|
|
});
|
|
|
|
app.post('/api/changeUserPermissions', optionalJwt, async (req, res) => {
|
|
const user_uid = req.body.user_uid;
|
|
const permission = req.body.permission;
|
|
const new_value = req.body.new_value;
|
|
|
|
if (!permission || !new_value) {
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
const success = await auth_api.changeUserPermissions(user_uid, permission, new_value);
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/changeRolePermissions', optionalJwt, async (req, res) => {
|
|
const role = req.body.role;
|
|
const permission = req.body.permission;
|
|
const new_value = req.body.new_value;
|
|
|
|
if (!permission || !new_value) {
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
const success = await auth_api.changeRolePermissions(role, permission, new_value);
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
// notifications
|
|
|
|
app.post('/api/getNotifications', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const notifications = await db_api.getRecords('notifications', {user_uid: uuid});
|
|
|
|
res.send({notifications: notifications});
|
|
});
|
|
|
|
// set notifications to read
|
|
app.post('/api/setNotificationsToRead', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await db_api.updateRecords('notifications', {user_uid: uuid}, {read: true});
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/deleteNotification', optionalJwt, async (req, res) => {
|
|
const uid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await db_api.removeRecord('notifications', {uid: uid});
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
|
|
const uuid = req.isAuthenticated() ? req.user.uid : null;
|
|
|
|
const success = await db_api.removeAllRecords('notifications', {user_uid: uuid});
|
|
|
|
res.send({success: success});
|
|
});
|
|
|
|
app.post('/api/telegramRequest', async (req, res) => {
|
|
if (!req.body.message && !req.body.message.text) {
|
|
logger.error('Invalid Telegram request received!');
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
const text = req.body.message.text;
|
|
const regex_exp = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)?/gi;
|
|
const url_regex = new RegExp(regex_exp);
|
|
const matched_urls = text.match(url_regex);
|
|
if (matched_urls && matched_urls.length) {
|
|
let parsed_url = null;
|
|
try {
|
|
parsed_url = new URL(matched_urls[0]);
|
|
} catch {
|
|
parsed_url = null;
|
|
}
|
|
|
|
if (!parsed_url || (parsed_url.protocol !== 'http:' && parsed_url.protocol !== 'https:')) {
|
|
logger.error('Invalid Telegram request received! URL protocol is not allowed.');
|
|
res.sendStatus(400);
|
|
return;
|
|
}
|
|
|
|
downloader_api.createDownload(parsed_url.toString(), 'video', {}, req.query.user_uid ? req.query.user_uid : null);
|
|
res.sendStatus(200);
|
|
} else {
|
|
logger.error('Invalid Telegram request received! Make sure you only send a valid URL.');
|
|
notifications_api.sendTelegramNotification({title: 'Invalid Telegram Request', body: 'Make sure you only send a valid URL.', url: text});
|
|
res.sendStatus(400);
|
|
}
|
|
});
|
|
|
|
// rss feed
|
|
|
|
app.get('/api/rss', async function (req, res) {
|
|
if (!config_api.getConfigItem('ytdl_enable_rss_feed')) {
|
|
logger.error('RSS feed is disabled! It must be enabled in the settings before it can be generated.');
|
|
res.sendStatus(403);
|
|
return;
|
|
}
|
|
|
|
// these are returned
|
|
const sort = req.query.sort ? JSON.parse(decodeURIComponent(req.query.sort)) : {by: 'registered', order: -1};
|
|
const range = req.query.range ? req.query.range.map(range_num => parseInt(range_num)) : null;
|
|
const text_search = req.query.text_search ? decodeURIComponent(req.query.text_search) : null;
|
|
const file_type_filter = req.query.file_type_filter;
|
|
const favorite_filter = req.query.favorite_filter === 'true';
|
|
const sub_id = req.query.sub_id ? decodeURIComponent(req.query.sub_id) : null;
|
|
const uuid = req.query.uuid ? decodeURIComponent(req.query.uuid) : null;
|
|
|
|
const {files} = await files_api.getAllFiles(sort, range, text_search, file_type_filter, favorite_filter, sub_id, uuid);
|
|
|
|
const { Feed } = await import('feed');
|
|
const feed = new Feed({
|
|
title: 'Downloads',
|
|
description: 'YoutubeDL-Material downloads',
|
|
id: utils.getBaseURL(),
|
|
link: utils.getBaseURL(),
|
|
image: 'https://github.com/voc0der/YoutubeDL-Material/blob/master/src/assets/images/logo_128px.png',
|
|
favicon: 'https://raw.githubusercontent.com/voc0der/YoutubeDL-Material/master/src/favicon.ico',
|
|
generator: 'YoutubeDL-Material'
|
|
});
|
|
|
|
files.forEach(file => {
|
|
feed.addItem({
|
|
title: file.title,
|
|
link: `${utils.getBaseURL()}/#/player;uid=${file.uid}`,
|
|
description: file.description,
|
|
author: [
|
|
{
|
|
name: file.uploader,
|
|
link: file.url
|
|
}
|
|
],
|
|
contributor: [],
|
|
date: file.timestamp,
|
|
// https://stackoverflow.com/a/45415677/8088021
|
|
image: escapeXmlEntities(file.thumbnailURL)
|
|
});
|
|
});
|
|
res.send(feed.rss2());
|
|
});
|
|
|
|
// web server
|
|
|
|
app.use(function(req, res, next) {
|
|
if (!oidc_api.isEnabled() || !config_api.getConfigItem('ytdl_multi_user_mode')) {
|
|
return next();
|
|
}
|
|
|
|
if (req.path.includes('/api/')) {
|
|
return next();
|
|
}
|
|
|
|
// Hash routes (/#/...) are received as "/" on the server. Let the SPA load so
|
|
// client-side routing can preserve return targets like /player;uid=... .
|
|
if (req.path === '/') {
|
|
return next();
|
|
}
|
|
|
|
const accept = req.accepts('html', 'json', 'xml');
|
|
if (accept !== 'html') {
|
|
return next();
|
|
}
|
|
|
|
const ext = path.extname(req.path);
|
|
if (ext !== '') {
|
|
return next();
|
|
}
|
|
|
|
const cookie_header = req.headers.cookie || '';
|
|
const has_bootstrap_cookie = cookie_header.split(';').map(cookie => cookie.trim()).includes('ytdl_oidc_bootstrap=1');
|
|
if (has_bootstrap_cookie) {
|
|
res.setHeader('Set-Cookie', 'ytdl_oidc_bootstrap=; Path=/; Max-Age=0; HttpOnly; SameSite=Lax');
|
|
return next();
|
|
}
|
|
|
|
const return_to = req.path && req.path !== '/' ? req.path : '/home';
|
|
const redirect_path = `/api/auth/oidc/login?returnTo=${encodeURIComponent(return_to)}`;
|
|
return res.redirect(redirect_path);
|
|
});
|
|
|
|
app.use(function(req, res, next) {
|
|
//if the request is not html then move along
|
|
var accept = req.accepts('html', 'json', 'xml');
|
|
if (accept !== 'html') {
|
|
return next();
|
|
}
|
|
|
|
// if the request has a '.' assume that it's for a file, move along
|
|
var ext = path.extname(req.path);
|
|
if (ext !== '') {
|
|
return next();
|
|
}
|
|
|
|
let index_path = path.join(__dirname, 'public', 'index.html');
|
|
|
|
res.setHeader('Content-Type', 'text/html');
|
|
|
|
fs.createReadStream(index_path).pipe(res);
|
|
|
|
});
|
|
|
|
let public_dir = path.join(__dirname, 'public');
|
|
|
|
app.use(express.static(public_dir));
|