diff --git a/Public API v1.yaml b/Public API v1.yaml index 9d61076..951c27d 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -1632,6 +1632,9 @@ components: $ref: '#/components/schemas/FileType' cropFileSettings: $ref: '#/components/schemas/CropFileSettings' + ignoreArchive: + type: boolean + description: If using youtube-dl archive, download will ignore it DownloadResponse: type: object properties: @@ -2080,17 +2083,12 @@ components: type: $ref: '#/components/schemas/FileType' DownloadArchiveRequest: - required: - - sub type: object properties: - sub: - required: - - archive_dir - type: object - properties: - archive_dir: - type: string + type: + $ref: '#/components/schemas/FileType' + sub_id: + type: string UpdaterStatus: required: - details diff --git a/backend/app.js b/backend/app.js index 66d896f..3c8e022 100644 --- a/backend/app.js +++ b/backend/app.js @@ -522,9 +522,6 @@ async function loadConfig() { db_api.database_initialized = true; db_api.database_initialized_bs.next(true); - // creates archive path if missing - await fs.ensureDir(utils.getArchiveFolder()); - // check migrations await checkMigrations(); @@ -830,7 +827,8 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) { youtubeUsername: req.body.youtubeUsername, youtubePassword: req.body.youtubePassword, ui_uid: req.body.ui_uid, - cropFileSettings: req.body.cropFileSettings + cropFileSettings: req.body.cropFileSettings, + ignoreArchive: req.body.ignoreArchive }; const download = await downloader_api.createDownload(url, type, options, user_uid); @@ -1517,15 +1515,18 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => { }); app.post('/api/downloadArchive', optionalJwt, async (req, res) => { - let sub = req.body.sub; - let archive_dir = sub.archive; + const uuid = req.isAuthenticated() ? req.user.uid : null; + const sub_id = req.body.sub_id; + const type = req.body.type; - let full_archive_path = path.join(archive_dir, 'archive.txt'); + const archive_text = await archive_api.generateArchive(type, uuid, sub_id); - if (await fs.pathExists(full_archive_path)) { - res.sendFile(full_archive_path); + if (archive_text !== null && archive_text !== undefined) { + res.setHeader('Content-type', "application/octet-stream"); + res.setHeader('Content-disposition', 'attachment; filename=archive.txt'); + res.send(archive_text); } else { - res.sendStatus(404); + res.sendStatus(400); } }); diff --git a/backend/archive.js b/backend/archive.js index 545c8cc..d7e1dde 100644 --- a/backend/archive.js +++ b/backend/archive.js @@ -3,20 +3,27 @@ const fs = require('fs-extra'); const db_api = require('./db'); -exports.generateArchive = async (user_uid = null, sub_id = null) => { - const archive_items = await db_api.getRecords('archives', {user_uid: user_uid, sub_id: sub_id}); +exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => { + const filter = {user_uid: user_uid, sub_id: sub_id}; + if (type) filter['type'] = type; + const archive_items = await db_api.getRecords('archives', filter); const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`); return archive_item_lines.join('\n'); } exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => { const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id); - const success = await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}, type: type}); + const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type}); + return success; +} + +exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => { + const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id}); return success; } exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => { - const archive_item = await db_api.getRecord('archives', {'key.extractor': extractor, 'key.id': id, type: type, user_uid: user_uid, sub_id: sub_id}); + const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id}); return !!archive_item; } @@ -37,7 +44,7 @@ exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = // we can't do a bulk write because we need to avoid duplicate archive items existing in db const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id); - await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}}); + await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id}); archive_import_count++; } return archive_import_count; @@ -71,10 +78,8 @@ exports.importArchives = async () => { const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => { return { - key: { - extractor: extractor, - id: id - }, + extractor: extractor, + id: id, type: type, user_uid: user_uid ? user_uid : null, sub_id: sub_id ? sub_id : null diff --git a/backend/db.js b/backend/db.js index 9811979..26663be 100644 --- a/backend/db.js +++ b/backend/db.js @@ -64,8 +64,7 @@ const tables = { primary_key: 'uid' }, archives: { - name: 'archives', - primary_key: 'key' + name: 'archives' }, test: { name: 'test' @@ -510,16 +509,22 @@ exports.deleteFile = async (uid, blacklistMode = false) => { let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); if (useYoutubeDLArchive) { - const archive_path = utils.getArchiveFolder(type, file_obj.user_uid, file_obj.sub_id ? (await exports.getRecord('subscriptions', {id: file_obj.sub_id})) : null); - - // get ID from JSON - - var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath)); - let id = null; - if (jsonobj) id = jsonobj.id; + // get id/extractor from JSON + + const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath)); + let retrievedID = null; + let retrievedExtractor = null; + if (info_json) { + retrievedID = info_json['id']; + retrievedExtractor = info_json['extractor']; + } // Remove file ID from the archive file, and write it to the blacklist (if enabled) - await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode); + if (!blacklistMode) { + // workaround until a files_api is created (using archive_api would make a circular dependency) + await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id}); + // await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id); + } } if (jsonExists) await fs.unlink(jsonPath); diff --git a/backend/downloader.js b/backend/downloader.js index cb2449c..30f1c3a 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -15,6 +15,7 @@ const categories_api = require('./categories'); const utils = require('./utils'); const db_api = require('./db'); const notifications_api = require('./notifications'); +const archive_api = require('./archive'); const mutex = new Mutex(); let should_check_downloads = true; @@ -202,6 +203,20 @@ async function collectInfo(download_uid) { return; } + const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive && !options.ignoreArchive) { + const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']); + if (exists_in_archive) { + const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`; + logger.warn(error); + if (download_uid) { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + await handleDownloadError(download, error); + return; + } + } + } + let category = null; // check if it fits into a category. If so, then get info again using new args @@ -353,19 +368,14 @@ async function downloadQueuedFile(download_uid) { // registers file in DB const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); + const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, download['user_uid'], download['sub_id']); + notifications_api.sendDownloadNotification(file_obj, download['user_uid']); file_objs.push(file_obj); } - if (options.merged_string !== null && options.merged_string !== undefined) { - const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']); - const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8'); - const diff = current_merged_archive.replace(options.merged_string, ''); - const archive_path = path.join(archive_folder, `archive_${type}.txt`); - fs.appendFileSync(archive_path, diff); - } - let container = null; if (file_objs.length > 1) { @@ -478,28 +488,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); } - let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive) { - const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid); - const archive_path = path.join(archive_folder, `archive_${type}.txt`); - - await fs.ensureDir(archive_folder); - await fs.ensureFile(archive_path); - - const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`); - await fs.ensureFile(blacklist_path); - - const merged_path = path.join(archive_folder, `merged_${type}.txt`); - await fs.ensureFile(merged_path); - // merges blacklist and regular archive - let inputPathList = [archive_path, blacklist_path]; - await mergeFiles(inputPathList, merged_path); - - options.merged_string = await fs.readFile(merged_path, "utf8"); - - downloadConfig.push('--download-archive', merged_path); - } - if (config_api.getConfigItem('ytdl_include_thumbnail')) { downloadConfig.push('--write-thumbnail'); } @@ -651,13 +639,3 @@ exports.generateNFOFile = (info, output_path) => { const xml = doc.end({ prettyPrint: true }); fs.writeFileSync(output_path, xml); } - -function getArchiveFolder(fileFolderPath, options, user_uid) { - if (options.customArchivePath) { - return path.join(options.customArchivePath); - } else if (user_uid) { - return path.join(fileFolderPath, 'archives'); - } else { - return path.join('appdata', 'archives'); - } -} \ No newline at end of file diff --git a/backend/subscriptions.js b/backend/subscriptions.js index d89a87c..f1d89e6 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -3,6 +3,7 @@ const path = require('path'); const youtubedl = require('youtube-dl'); const config_api = require('./config'); +const archive_api = require('./archive'); const utils = require('./utils'); const logger = require('./logger'); @@ -160,10 +161,10 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, let basePath = null; basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions') : config_api.getConfigItem('ytdl_subscriptions_base_path'); - const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); const appendedBasePath = getAppendedBasePath(sub, basePath); const name = file; let retrievedID = null; + let retrievedExtractor = null; await db_api.removeRecord('files', {uid: file_uid}); @@ -182,7 +183,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, ]); if (jsonExists) { - retrievedID = fs.readJSONSync(jsonPath)['id']; + const info_json = fs.readJSONSync(jsonPath); + retrievedID = info_json['id']; + retrievedExtractor = info_json['extractor']; await fs.unlink(jsonPath); } @@ -200,11 +203,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, return false; } else { // check if the user wants the video to be redownloaded (deleteForever === false) - if (useArchive && retrievedID) { - const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub); - - // Remove file ID from the archive file, and write it to the blacklist (if enabled) - await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever); + const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useArchive && !deleteForever) { + await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id); } return true; } @@ -335,8 +336,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de else basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); - const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - let appendedBasePath = getAppendedBasePath(sub, basePath); const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; @@ -372,21 +371,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push(...customArgsArray); } - let archive_dir = null; - let archive_path = null; - - if (useArchive && !redownload) { - if (sub.archive) { - archive_dir = sub.archive; - if (sub.type && sub.type === 'audio') { - archive_path = path.join(archive_dir, 'merged_audio.txt'); - } else { - archive_path = path.join(archive_dir, 'merged_video.txt'); - } - } - downloadConfig.push('--download-archive', archive_path); - } - if (sub.timerange && !redownload) { downloadConfig.push('--dateafter', sub.timerange); } @@ -429,7 +413,14 @@ async function getFilesToDownload(sub, output_jsons) { if (file_with_path_exists) { // or maybe just overwrite??? logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`) + continue; } + const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); + if (useYoutubeDLArchive) { + const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id); + if (exists_in_archive) continue; + } + files_to_download.push(output_json); } } diff --git a/backend/test/tests.js b/backend/test/tests.js index 1080ff3..f0050b1 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -625,6 +625,7 @@ describe('Tasks', function() { describe('Archive', async function() { beforeEach(async function() { + await db_api.connectToDB(); await db_api.removeAllRecords('archives', {user_uid: 'test_user'}); }); @@ -645,25 +646,50 @@ describe('Archive', async function() { const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'}); console.log(archive_items); assert(archive_items.length === 4); - assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor2').length === 1); - assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor1').length === 3); + assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1); + assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3); const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'}); assert(success); }); it('Get archive', async function() { - await archive_api.addToArchive('testextractor1', 'video', 'testing1', 'test_user'); - await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user'); - await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user'); + await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user'); + await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user'); - const archive_item1 = await db_api.getRecord('archives', {key: {extractor: 'testextractor1', id: 'testing1'}}); - const archive_item2 = await db_api.getRecord('archives', {key: {extractor: 'testextractor2', id: 'testing1'}}); + const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'}); + const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'}); assert(archive_item1 && archive_item2); + }); + + it('Archive duplicates', async function() { + await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user'); + await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user'); + await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user'); + + await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user'); + + const count = await db_api.getRecords('archives', {id: 'testing1'}, true); + assert(count === 3); + }); + + it('Remove from archive', async function() { + await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user'); + await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user'); + await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user'); + + const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user'); + assert(success); + + const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'}); + assert(!!archive_item1); + + const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'}); + assert(!archive_item2); - const count = await db_api.getRecords('archives', {key: {id: 'testing1'}}, true); - assert(count === 2); + const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'}); + assert(!!archive_item3); }); }); diff --git a/src/api-types/models/DownloadArchiveRequest.ts b/src/api-types/models/DownloadArchiveRequest.ts index dad4328..6123a30 100644 --- a/src/api-types/models/DownloadArchiveRequest.ts +++ b/src/api-types/models/DownloadArchiveRequest.ts @@ -2,8 +2,9 @@ /* tslint:disable */ /* eslint-disable */ +import type { FileType } from './FileType'; + export type DownloadArchiveRequest = { - sub: { -archive_dir: string; -}; + type?: FileType; + sub_id?: string; }; \ No newline at end of file diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 3a34612..4cda439 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -434,8 +434,8 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'uploadCookies', fileFormData, this.httpOptions); } - downloadArchive(sub) { - const body: DownloadArchiveRequest = {sub: sub}; + downloadArchive(type: FileType, sub_id: string) { + const body: DownloadArchiveRequest = {type: type, sub_id: sub_id}; return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params}); }