Archive text file is now temporarily added to a sub dir when checking a sub for speed purposes

backend/subscriptions.js

@@ -229,13 +229,20 @@ async function getVideosForSub(sub, user_uid = null) {
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
-    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
+    logger.verbose(`Subscription: getting list of videos to download for ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
 
+            // remove temporary archive file if it exists
+            const archive_path = path.join(appendedBasePath, 'archive.txt');
+            const archive_exists = await fs.pathExists(archive_path);
+            if (archive_exists) {
+                await fs.unlink(archive_path);
+            }
+
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
                 logger.error(err.stderr ? err.stderr : err.message);
@@ -354,6 +361,16 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
 
     downloadConfig.push(...qualityPath)
 
+    // if archive is being used, we want to quickly skip videos that are in the archive. otherwise sub download can be permanently slow (vs. just the first time)
+    const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
+    if (useYoutubeDLArchive) {
+        const archive_text = await archive_api.generateArchive(sub.type, sub.user_uid, sub.id);
+        logger.verbose(`Generating temporary archive file for ${sub.id} with ${archive_text.split('\n').length - 1} entries.`)
+        const archive_path = path.join(appendedBasePath, 'archive.txt');
+        await fs.writeFile(archive_path, archive_text);
+        downloadConfig.push('--download-archive', archive_path);
+    }
+
     if (sub.custom_args) {
         const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {