Merge pull request #602 from Tzahi12345/heroku-fixes-and-repo-cleanup

Various bug fixes and heroku improvements

Dockerfile

@@ -64,6 +64,9 @@ RUN npm config set strict-ssl false && \
   npm install pm2 -g && \
   npm install && chown -R $UID:$GID ./
 
+# needed for ubuntu, see #596
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
 COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ]
 COPY --chown=$UID:$GID [ "/backend/", "/app/" ]
 

Dockerfile.heroku

@@ -0,0 +1,2 @@
+FROM tzahi12345/youtubedl-material:nightly
+CMD [ "pm2-runtime", "pm2.config.js" ]

Procfile

@@ -1 +0,0 @@
-web: npm start --prefix backend

app.json

@@ -2,6 +2,7 @@
     "name": "YoutubeDL-Material",
     "description": "An open-source and self-hosted YouTube downloader based on Google's Material Design specifications.",
     "repository": "https://github.com/Tzahi12345/YoutubeDL-Material",
+    "stack": "container",
     "logo": "https://i.imgur.com/GPzvPiU.png",
     "keywords": ["youtube-dl", "youtubedl-material", "nodejs"]
   }

backend/app.js

@@ -249,14 +249,6 @@ async function startServer() {
     });
 }
 
-async function restartServer(is_update = false) {
-    logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
-
-    // the following line restarts the server through nodemon
-    fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
-    process.exit(1);
-}
-
 async function updateServer(tag) {
     // no tag provided means update to the latest version
     if (!tag) {
@@ -297,7 +289,7 @@ async function updateServer(tag) {
             updating: true,
             'details': 'Update complete! Restarting server...'
         }
-        restartServer(true);
+        utils.restartServer(true);
     }, err => {
         logger.error(err);
         updaterStatus = {
@@ -676,6 +668,7 @@ async function getUrlInfos(url) {
 
 async function startYoutubeDL() {
     // auto update youtube-dl
+    youtubedl_api.verifyBinaryExistsLinux();
     const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
     if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
 }
@@ -764,7 +757,7 @@ app.get('/api/versionInfo', (req, res) => {
 
 app.post('/api/restartServer', optionalJwt, (req, res) => {
     // delayed by a little bit so that the client gets a response
-    setTimeout(() => {restartServer()}, 100);
+    setTimeout(() => {utils.restartServer()}, 100);
     res.send({success: true});
 });
 
@@ -1802,6 +1795,7 @@ app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
 
 app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
     const backup_dir = path.join('appdata', 'db_backup');
+    fs.ensureDirSync(backup_dir);
     const db_backups = [];
 
     const candidate_backups = await utils.recFindByExt(backup_dir, 'bak', null, [], false);

backend/consts.js

@@ -222,4 +222,83 @@ exports.AVAILABLE_PERMISSIONS = [
 
 exports.DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details'
 
+// args that have a value after it (e.g. -o <output> or -f <format>)
+const YTDL_ARGS_WITH_VALUES = [
+    '--default-search',
+    '--config-location',
+    '--proxy',
+    '--socket-timeout',
+    '--source-address',
+    '--geo-verification-proxy',
+    '--geo-bypass-country',
+    '--geo-bypass-ip-block',
+    '--playlist-start',
+    '--playlist-end',
+    '--playlist-items',
+    '--match-title',
+    '--reject-title',
+    '--max-downloads',
+    '--min-filesize',
+    '--max-filesize',
+    '--date',
+    '--datebefore',
+    '--dateafter',
+    '--min-views',
+    '--max-views',
+    '--match-filter',
+    '--age-limit',
+    '--download-archive',
+    '-r',
+    '--limit-rate',
+    '-R',
+    '--retries',
+    '--fragment-retries',
+    '--buffer-size',
+    '--http-chunk-size',
+    '--external-downloader',
+    '--external-downloader-args',
+    '-a',
+    '--batch-file',
+    '-o',
+    '--output',
+    '--output-na-placeholder',
+    '--autonumber-start',
+    '--load-info-json',
+    '--cookies',
+    '--cache-dir',
+    '--encoding',
+    '--user-agent',
+    '--referer',
+    '--add-header',
+    '--sleep-interval',
+    '--max-sleep-interval',
+    '-f',
+    '--format',
+    '--merge-output-format',
+    '--sub-format',
+    '--sub-lang',
+    '-u',
+    '--username',
+    '-p',
+    '--password',
+    '-2',
+    '--twofactor',
+    '--video-password',
+    '--ap-mso',
+    '--ap-username',
+    '--ap-password',
+    '--audio-format',
+    '--audio-quality',
+    '--recode-video',
+    '--postprocessor-args',
+    '--metadata-from-title',
+    '--fixup',
+    '--ffmpeg-location',
+    '--exec',
+    '--convert-subs'
+];
+
+// we're using a Set here for performance
+exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
+
 exports.CURRENT_VERSION = 'v4.2';

backend/db.js

@@ -85,8 +85,6 @@ exports.initialize = (input_db, input_users_db) => {
 }
 
 exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
-    using_local_db = config_api.getConfigItem('ytdl_use_local_db'); // verify
-    if (using_local_db && !custom_connection_string) return;
     const success = await exports._connectToDB(custom_connection_string);
     if (success) return true;
 

backend/downloader.js

@@ -485,7 +485,7 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
         }
 
         if (options.additionalArgs && options.additionalArgs !== '') {
-            downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
+            downloadConfig = utils.injectArgs(downloadConfig, options.additionalArgs.split(',,'));
         }
 
         const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');

backend/test/tests.js

@@ -386,6 +386,21 @@ describe('Downloader', function() {
         assert(fs.existsSync(nfo_file_path), true);
         fs.unlinkSync(nfo_file_path);
     });
+
+    it('Inject args', async function() {
+        const original_args1 = ['--no-resize-buffer', '-o', '%(title)s', '--no-mtime'];
+        const new_args1 = ['--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
+        const updated_args1 = utils.injectArgs(original_args1, new_args1);
+        const expected_args1 = ['--no-resize-buffer', '--no-mtime', '--age-limit', '25', '--yes-playlist', '--abort-on-error', '-o', '%(id)s'];
+        assert(JSON.stringify(updated_args1), JSON.stringify(expected_args1));
+
+        const original_args2 = ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3'];
+        const new_args2 =  ['--add-metadata', '--embed-thumbnail', '--convert-thumbnails', 'jpg'];
+        const updated_args2 = utils.injectArgs(original_args2, new_args2);
+        const expected_args2 =  ['-o', '%(title)s.%(ext)s', '--write-info-json', '--print-json', '--audio-quality', '0', '-x', '--audio-format', 'mp3', '--add-metadata', '--embed-thumbnail', '--convert_thumbnails', 'jpg'];
+        console.log(updated_args2);
+        assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
+    });
 });
 
 describe('Tasks', function() {

backend/utils.js

@@ -415,6 +415,47 @@ async function fetchFile(url, path, file_label) {
     });
 }
 
+async function restartServer(is_update = false) {
+    logger.info(`${is_update ? 'Update complete! ' : ''}Restarting server...`);
+
+    // the following line restarts the server through nodemon
+    fs.writeFileSync(`restart${is_update ? '_update' : '_general'}.json`, 'internal use only');
+    process.exit(1);
+}
+
+// adds or replaces args according to the following rules:
+//  - if it already exists and has value, then replace both arg and value
+//  - if already exists and doesn't have value, ignore
+//  - if it doesn't exist and has value, add both arg and value
+//  - if it doesn't exist and doesn't have value, add arg
+function injectArgs(original_args, new_args) {
+    const updated_args = original_args.slice();
+    try {
+        for (let i = 0; i < new_args.length; i++) {
+            const new_arg = new_args[i];
+            if (!new_arg.startsWith('-') && !new_arg.startsWith('--') && i > 0 && original_args.includes(new_args[i - 1])) continue;
+            
+            if (CONSTS.YTDL_ARGS_WITH_VALUES.has(new_arg)) {
+                if (original_args.includes(new_arg)) {
+                    const original_index = original_args.indexOf(new_arg);
+                    original_args.splice(original_index, 2);
+                }
+
+                updated_args.push(new_arg, new_args[i + 1]);
+            } else {
+                if (!original_args.includes(new_arg)) {
+                    updated_args.push(new_arg);
+                }
+            }
+        }
+    } catch (err) {
+        logger.warn(err);
+        logger.warn(`Failed to inject args (${new_args}) into (${original_args})`);
+    }
+
+    return updated_args;
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -458,5 +499,7 @@ module.exports = {
     wait: wait,
     checkExistsWithTimeout: checkExistsWithTimeout,
     fetchFile: fetchFile,
+    restartServer: restartServer,
+    injectArgs: injectArgs,
     File: File
 }

backend/youtube-dl.js

@@ -6,6 +6,8 @@ const utils = require('./utils');
 const CONSTS = require('./consts');
 const config_api = require('./config.js');
 
+const OUTDATED_VERSION = "2020.00.00";
+
 const is_windows = process.platform === 'win32';
 
 const download_sources = {
@@ -31,7 +33,7 @@ exports.checkForYoutubeDLUpdate = async () => {
         let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
         if (!current_app_details_exists) {
             logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
-            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
+            fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version": OUTDATED_VERSION, "downloader": default_downloader});
         }
         let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
         let current_version = current_app_details['version'];
@@ -86,6 +88,18 @@ exports.updateYoutubeDL = async (latest_update_version) => {
     await download_sources[default_downloader]['func'](latest_update_version);
 }
 
+exports.verifyBinaryExistsLinux = () => {
+    const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
+    if (!is_windows && details_json && details_json['path'].includes('.exe')) {
+        details_json['path'] = 'node_modules/youtube-dl/bin/youtube-dl';
+        details_json['exec'] = 'youtube-dl';
+        details_json['version'] = OUTDATED_VERSION;
+        fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
+
+        utils.restartServer();
+    }
+}
+
 async function downloadLatestYoutubeDLBinary(new_version) {
     const file_ext = is_windows ? '.exe' : '';
 

chrome-extension.pem

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMX9Wk5SM5cIfY
-6ReKX3ybY1rsbNbOzG8ceN7yyeXB0mor8pVsX1MOna2HewOyBuaaYNJRO4tJBxic
-7a8zQErfgHL/i/QrVvVCpfJ7xKvq6zij5NYoqd/FBUwawqjeH5/voIcAp9z5Vmsr
-kL0sxJUKy6b4IWNp3noU7Nvq2RwxnXQbKDhz8FrX6oQAnDC6gsG5a2OSPsaE4oqw
-6nmonORJypmpP5hqyHY8ffXBT2lAxjHT7OnYbaCBe2TQP8+rH6rDBOhjVNtUJ089
-ocTQL6LtQEPkcF4yKJmtcOwHl8OPGZs5l9i8xb4j9RuSPkm2lbzZX8sOsdGGoqJZ
-q68nYhsHAgMBAAECggEAXmtKEzfPObq88B/kAcgSk+FngMHZzcmR7bgD3GwdSxnQ
-dkRI9zvk7eQ35tcUwntAr4Lat6/ILjFqlBmVLxrdXHuF5Xz9jcZLYgKzz61xdYM9
-dC6FKF0u5eGIIvbauGAo7jaeGFX1F3Zu5b4lP9kEOGwU1B7sxF0FzsQM5+dtCJgv
-We/hWQeF+9gtoVnkCSS/Mq2p0UomXXHW0Bz4+HuHlTR9aiYbviYnotABiLUhZyzt
-v5yUaktb9qniBfdLpRlq8cp06xYlTEA9gJpa4Pnok8OWUsbAiW6EiXUSaZ/cchVa
-AnO8WWYvVOnnt6WHI3+QdFTnqVjE5TBX4N/7bVhHGQKBgQD0dtbFqp7vZK/jVYvE
-z0WPdySOg2ZDmoSfk5ZlR1+Y9zWToHv0qu8zqoOjL8Ubxrh9fGlOow+cCVdkEuaa
-jWC2AWetuRvW0Z5A3XMXr0/N/1fgOkTqtp3WNrUPjVJahEg3lN+90opgFoT8swSi
-s1oxW0oLcVIlrjhGBXAPCfsAuQKBgQDWBLRhHsRAvGcK5wGuVnxVApTIyBOermsW
-3bJt+7+UI+4sYrBAwkWdQG93IG0cQtn48TEPBgmR2fjRF5IFT9M4/u+QOeeByT7I
-we7nVtHgSY5ByC9N0mjWbcmSg8fktz/LonjldNC4kWdOFb75fxGf8kOGS5rUaMA4
-zHucfB6ZvwKBgQCPHJrysMXGY21MaqIeHzEboaX3ABl37hdBzAa5V6UxSVdGCydF
-vmO2HVZey/JaJmWOoKyNaowSzq0oWqBBTg6VvhDR9JHFmoVId9uOvAS+FYN+Mt5x
-gWK5KuGoLxVNBC+6yh6JY526TrSfsrU+Aj0Es+qO9FIg2PL8muZVB4S3kQKBgH/5
-CDMaxpc/EQ5/2413wZjDllwI51J3USm3Hz6Mzp2ybnSz/lh60k2Zfg1polTH1Lb6
-4i7tmUNRZ2sAARyUAuWN64n+VeRRhe1dqZFDZPQMh7fmEAMk0fOGaoXlrt2ghdEq
-Mchi9Xun1nHmpu9hgBR4NNBU3RwuFuLfwvprbZDZAoGAWa62QJChE86xQGP1MrL2
-SbIzw3cfeP5xdQ3MKldJiy5IkbMR7Z13WZ7FwvPTy0g/onLHD1rqlm1kUMsGRHpD
-5vH06PNpKXQ6x8BYaRGtE6P39jLycO/X+WK/lYTrWo1bR+mGCebDh4B5XrwT3gI6
-x4Gvz134pZCTyQCf5JCwbQs=
------END PRIVATE KEY-----

heroku.yml

@@ -0,0 +1,3 @@
+build:
+  docker:
+    web: Dockerfile.heroku

package.json

@@ -12,7 +12,8 @@
     "lint": "ng lint",
     "e2e": "ng e2e",
     "electron": "ng build --base-href ./ && electron .",
-    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true"
+    "generate": "openapi --input ./\"Public API v1.yaml\" --output ./src/api-types --exportCore false --exportServices false --exportModels true",
+    "i18n-source": "ng extract-i18n --output-path=src/assets/i18n"
   },
   "engines": {
     "node": "12.3.1",

src/app/subscriptions/subscriptions.component.html

@@ -36,6 +36,9 @@
                 <ng-container i18n="Subscription playlist not available text">Name not available. Playlist retrieval in progress.</ng-container>
             </div>
         </a>
+        <button mat-icon-button (click)="editSubscription(sub)">
+            <mat-icon>edit</mat-icon>
+        </button>
         <button mat-icon-button (click)="showSubInfo(sub)">
             <mat-icon>info</mat-icon>
         </button>