added docker support

reworked backend to allow for containerization. config items can now be overwritten by environment vars

fixed bug during building

updated youtube-dl version in backend

Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y \
+  nodejs \
+  apache2 \
+  npm \
+  youtube-dl
+
+# Change directory so that our commands run inside this new directory
+WORKDIR /var/www/html
+
+# Copy dependency definitions
+COPY ./ /var/www/html/
+
+# Change directory to backend
+WORKDIR /var/www/html/backend
+
+# Install dependencies on backend
+RUN npm install
+
+# Change back to original directory
+WORKDIR /var/www/html
+
+# Expose the port the app runs in
+EXPOSE 80
+
+# Run the specified command within the container.
+CMD ./docker_wrapper.sh

backend/app.js

@@ -2,7 +2,6 @@ var async = require('async');
 var fs = require('fs');
 var path = require('path');
 var youtubedl = require('youtube-dl');
-var config = require('config');
 var https = require('https');
 var express = require("express");
 var bodyParser = require("body-parser");
@@ -10,6 +9,7 @@ var archiver = require('archiver');
 const low = require('lowdb')
 var URL = require('url').URL;
 const shortid = require('shortid')
+var config_api = require('./config.js'); 
 
 var app = express();
 
@@ -18,65 +18,56 @@ const adapter = new FileSync('db.json');
 const db = low(adapter)
 
 // Set some defaults
-db.defaults({ playlists: {
-    audio: [],
-    video: []
-}}).write();
-
+db.defaults(
+    { 
+        playlists: {
+            audio: [],
+            video: []
+        },
+        configWriteFlag: false
+}).write();
+
+// config values
+var frontendUrl = null;
+var backendUrl = null;
+var backendPort = 17442;
+var usingEncryption = null;
+var basePath = null;
+var audioFolderPath = null;
+var videoFolderPath = null;
+var downloadOnlyMode = null;
+var useDefaultDownloadingAgent = null;
+var customDownloadingAgent = null;
+
+// other needed values
+var options = null; // encryption options
+var url_domain = null;
 
 // check if debug mode
 let debugMode = process.env.YTDL_MODE === 'debug';
 
 if (debugMode) console.log('YTDL-Material in debug mode!');
 
-var frontendUrl = !debugMode ? config.get("YoutubeDLMaterial.Host.frontendurl") : 'http://localhost:4200';
-var backendUrl = config.get("YoutubeDLMaterial.Host.backendurl")
-var backendPort = 17442;
-var usingEncryption = config.get("YoutubeDLMaterial.Encryption.use-encryption");
-var basePath = config.get("YoutubeDLMaterial.Downloader.path-base");
-var audioFolderPath = config.get("YoutubeDLMaterial.Downloader.path-audio");
-var videoFolderPath = config.get("YoutubeDLMaterial.Downloader.path-video");
-var downloadOnlyMode = config.get("YoutubeDLMaterial.Extra.download_only_mode")
-var useDefaultDownloadingAgent = config.get("YoutubeDLMaterial.Advanced.use_default_downloading_agent");
-var customDownloadingAgent = config.get("YoutubeDLMaterial.Advanced.custom_downloading_agent");
 var validDownloadingAgents = [
     'aria2c'
 ]
-if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) {
-    console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`)
-}
-
-var descriptors = {};
-
-
-if (usingEncryption)
-{
 
-    var certFilePath = path.resolve(config.get("YoutubeDLMaterial.Encryption.cert-file-path"));
-    var keyFilePath = path.resolve(config.get("YoutubeDLMaterial.Encryption.key-file-path"));
+// don't overwrite config if it already happened.. NOT
+// let alreadyWritten = db.get('configWriteFlag').value();
+let writeConfigMode = process.env.write_ytdl_config;
+var config = null;
 
-    var certKeyFile = fs.readFileSync(keyFilePath);
-    var certFile = fs.readFileSync(certFilePath);
-
-    var options = {
-        key: certKeyFile,
-        cert: certFile
-    };
+if (writeConfigMode) {
+    setAndLoadConfig();
+} else {
+    loadConfig();
 }
 
-
+var descriptors = {};
 
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
 
-var url_domain = new URL(frontendUrl);
-
-app.use(function(req, res, next) {
-    res.header("Access-Control-Allow-Origin", url_domain.origin);
-    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
-    next();
-});
-
 app.get('/using-encryption', function(req, res) {
     res.send(usingEncryption);
     res.end("yes");
@@ -94,6 +85,109 @@ function File(id, title, thumbnailURL, isAudio, duration) {
 
 // actual functions
 
+function startServer() {
+    if (usingEncryption)
+    {
+        https.createServer(options, app).listen(backendPort, function() {
+            console.log('HTTPS: Anchor set on 17442');
+        });
+    }
+    else
+    {
+        app.listen(backendPort,function(){
+            console.log("HTTP: Started on PORT " + backendPort);
+        });
+    }
+}
+
+async function setAndLoadConfig() {
+    await setConfigFromEnv();
+    await loadConfig();
+    // console.log(backendUrl);
+}
+
+async function setConfigFromEnv() {
+    return new Promise(resolve => {
+        let config_items = getEnvConfigItems();
+        let success = config_api.setConfigItems(config_items);
+        if (success) {
+            console.log('Config items set using ENV variables.');
+            setTimeout(() => resolve(true), 100);
+        } else {
+            console.log('ERROR: Failed to set config items using ENV variables.');
+            resolve(false);
+        }
+    });
+}
+
+async function loadConfig() {
+    return new Promise(resolve => {
+        // get config library
+        // config = require('config');
+
+        frontendUrl = !debugMode ? config_api.getConfigItem('ytdl_frontend_url') : 'http://localhost:4200';
+        backendUrl = config_api.getConfigItem('ytdl_backend_url')
+        backendPort = 17442;
+        usingEncryption = config_api.getConfigItem('ytdl_use_encryption');
+        basePath = config_api.getConfigItem('ytdl_base_path');
+        audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
+        videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
+        downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode');
+        useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
+        customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
+        if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) {
+            console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`)
+        }
+
+        if (usingEncryption)
+        {
+            var certFilePath = path.resolve(config_api.getConfigItem('ytdl_cert_file_path'));
+            var keyFilePath = path.resolve(config_api.getConfigItem('ytdl_key_file_path'));
+
+            var certKeyFile = fs.readFileSync(keyFilePath);
+            var certFile = fs.readFileSync(certFilePath);
+
+            options = {
+                key: certKeyFile,
+                cert: certFile
+            };
+        }
+
+        url_domain = new URL(frontendUrl);
+
+        // start the server here
+        startServer();
+
+        resolve(true);
+    });
+    
+}
+
+function getOrigin() {
+    return url_domain.origin;
+}
+
+// gets a list of config items that are stored as an environment variable
+function getEnvConfigItems() {
+    let config_items = [];
+
+    let config_item_keys = Object.keys(config_api.CONFIG_ITEMS);
+    for (let i = 0; i < config_item_keys.length; i++) {
+        let key = config_item_keys[i];
+        if (process['env'][key]) {
+            const config_item = generateEnvVarConfigItem(key);
+            config_items.push(config_item);
+        }
+    }
+    
+    return config_items;
+}
+
+// gets value of a config item and stores it in an object
+function generateEnvVarConfigItem(key) {
+    return {key: key, value: process['env'][key]};
+}
+
 function getThumbnailMp3(name)
 {
     var obj = getJSONMp3(name);
@@ -382,6 +476,12 @@ async function getUrlInfos(urls) {
     });
 }
 
+app.use(function(req, res, next) {
+    res.header("Access-Control-Allow-Origin", getOrigin());
+    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+    next();
+});
+
 app.post('/tomp3', function(req, res) {
     var url = req.body.url;
     var date = Date.now();
@@ -879,19 +979,3 @@ app.get('/audio/:id', function(req , res){
         success: !!result
     })
 });
-
-
-
-
-if (usingEncryption)
-{
-    https.createServer(options, app).listen(backendPort, function() {
-        console.log('HTTPS: Anchor set on 17442');
-    });
-}
-else
-{
-    app.listen(backendPort,function(){
-        console.log("HTTP: Started on PORT " + backendPort);
-    });
-}

backend/config.js

@@ -0,0 +1,112 @@
+const fs = require('fs');
+
+let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
+
+let configPath = 'config/default.json';
+
+// https://stackoverflow.com/questions/6491463/accessing-nested-javascript-objects-with-string-key
+Object.byString = function(o, s) {
+    s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
+    s = s.replace(/^\./, '');           // strip a leading dot
+    var a = s.split('.');
+    for (var i = 0, n = a.length; i < n; ++i) {
+        var k = a[i];
+        if (k in o) {
+            o = o[k];
+        } else {
+            return;
+        }
+    }
+    return o;
+}
+
+function getParentPath(path) {
+    let elements = path.split('.');
+    elements.splice(elements.length - 1, 1);
+    return elements.join('.');
+}
+
+function getElementNameInConfig(path) {
+    let elements = path.split('.');
+    return elements[elements.length - 1];
+}
+
+/*
+* Gets config file and returns as a json
+*/
+function getConfigFile() {
+    let raw_data = fs.readFileSync(configPath);
+    try {
+        let parsed_data = JSON.parse(raw_data);
+        return parsed_data;
+    } catch(e) {
+        console.log('ERROR: Failed to get config file');
+        return null;
+    }
+}
+
+function setConfigFile(config) {
+    try {
+        fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+        return true;
+    } catch(e) {
+        return false;
+    }
+}
+
+function getConfigItem(key) {
+    let config_json = getConfigFile();
+    if (!CONFIG_ITEMS[key]) console.log('cannot find config with key ' + key);
+    let path = CONFIG_ITEMS[key]['path'];
+    return Object.byString(config_json, path);
+};
+
+function setConfigItem(key, value) {
+    let success = false;
+    let config_json = getConfigFile();
+    let path = CONFIG_ITEMS[key]['path'];
+    let parent_path = getParentPath(path);
+    let element_name = getElementNameInConfig(path);
+    
+    let parent_object = Object.byString(config_json, parent_path);
+    if (value === 'false' || value === 'true') {
+        parent_object[element_name] = (value === 'true');
+    } else {
+        parent_object[element_name] = value;
+    }
+
+    success = setConfigFile(config_json);
+
+    return success;
+};
+
+function setConfigItems(items) {
+    let success = false;
+    let config_json = getConfigFile();
+    for (let i = 0; i < items.length; i++) {
+        let key = items[i].key;
+        let value = items[i].value;
+
+        // if boolean strings, set to booleans again
+        if (value === 'false' || value === 'true') {
+            value = (value === 'true');
+        }
+
+        let item_path = CONFIG_ITEMS[key]['path'];
+        let item_parent_path = getParentPath(item_path);
+        let item_element_name = getElementNameInConfig(item_path);
+        
+        let item_parent_object = Object.byString(config_json, item_parent_path);
+        item_parent_object[item_element_name] = value;
+    }
+
+    success = setConfigFile(config_json);
+    return success;
+}
+
+module.exports = {
+    getConfigItem: getConfigItem,
+    setConfigItem: setConfigItem,
+    setConfigItems: setConfigItems,
+    CONFIG_ITEMS: CONFIG_ITEMS
+}

backend/consts.js

@@ -0,0 +1,91 @@
+var config = require('config');
+
+let CONFIG_ITEMS = {
+    // Host
+    'ytdl_frontend_url': {
+        'key': 'ytdl_frontend_url',
+        'path': 'YoutubeDLMaterial.Host.frontendurl'
+    },
+    'ytdl_backend_url': {
+        'key': 'ytdl_backend_url',
+        'path': 'YoutubeDLMaterial.Host.backendurl'
+    },
+
+    // Encryption
+    'ytdl_use_encryption': {
+        'key': 'ytdl_use_encryption',
+        'path': 'YoutubeDLMaterial.Encryption.use-encryption'
+    },
+    'ytdl_cert_file_path': {
+        'key': 'ytdl_cert_file_path',
+        'path': 'YoutubeDLMaterial.Encryption.cert-file-path'
+    },
+    'ytdl_key_file_path': {
+        'key': 'ytdl_key_file_path',
+        'path': 'YoutubeDLMaterial.Encryption.key-file-path'
+    },
+
+    // Downloader
+    'ytdl_base_path': {
+        'key': 'ytdl_base_path',
+        'path': 'YoutubeDLMaterial.Downloader.path-base'
+    },
+    'ytdl_audio_folder_path': {
+        'key': 'ytdl_audio_folder_path',
+        'path': 'YoutubeDLMaterial.Downloader.path-audio'
+    },
+    'ytdl_video_folder_path': {
+        'key': 'ytdl_video_folder_path',
+        'path': 'YoutubeDLMaterial.Downloader.path-video'
+    },
+
+    // Extra
+    'ytdl_title_top': {
+        'key': 'ytdl_title_top',
+        'path': 'YoutubeDLMaterial.Extra.title_top'
+    },
+    'ytdl_file_manager_enabled': {
+        'key': 'ytdl_file_manager_enabled',
+        'path': 'YoutubeDLMaterial.Extra.file_manager_enabled'
+    },
+    'ytdl_allow_quality_select': {
+        'key': 'ytdl_allow_quality_select',
+        'path': 'YoutubeDLMaterial.Extra.allow_quality_select'
+    },
+    'ytdl_download_only_mode': {
+        'key': 'ytdl_download_only_mode',
+        'path': 'YoutubeDLMaterial.Extra.download_only_mode'
+    },
+
+    // API
+    'ytdl_use_youtube_api': {
+        'key': 'ytdl_use_youtube_api',
+        'path': 'YoutubeDLMaterial.API.use_youtube_API'
+    },
+    'ytdl_youtube_api_key': {
+        'key': 'ytdl_youtube_api_key',
+        'path': 'YoutubeDLMaterial.API.youtube_API_key'
+    },
+
+    // Themes
+    'ytdl_default_theme': {
+        'key': 'ytdl_default_theme',
+        'path': 'YoutubeDLMaterial.Themes.default_theme'
+    },
+    'ytdl_allow_theme_change': {
+        'key': 'ytdl_allow_theme_change',
+        'path': 'YoutubeDLMaterial.Themes.allow_theme_change'
+    },
+
+    // Advanced
+    'ytdl_use_default_downloading_agent': {
+        'key': 'ytdl_use_default_downloading_agent',
+        'path': 'YoutubeDLMaterial.Advanced.use_default_downloading_agent'
+    },
+    'ytdl_custom_downloading_agent': {
+        'key': 'ytdl_custom_downloading_agent',
+        'path': 'YoutubeDLMaterial.Advanced.custom_downloading_agent'
+    },
+};
+
+module.exports.CONFIG_ITEMS = CONFIG_ITEMS;

backend/package.json

@@ -4,7 +4,8 @@
   "description": "backend for YoutubeDL-Material",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node app.js"
   },
   "repository": {
     "type": "git",
@@ -24,6 +25,6 @@
     "express": "^4.17.1",
     "lowdb": "^1.0.0",
     "shortid": "^2.2.15",
-    "youtube-dl": "^2.3.0"
+    "youtube-dl": "^3.0.2"
   }
 }

docker-compose.yml

@@ -0,0 +1,33 @@
+
+version: "3.2"
+services:
+    ytdl_material:
+        build: .
+        environment: 
+            # config items
+            ytdl_frontend_url: http://localhost:8998
+            ytdl_backend_url: http://localhost:17442/
+            ytdl_use_encryption: 'false'
+            ytdl_cert_file_path: /etc/letsencrypt/live/example.com/fullchain.pem
+            ytdl_key_file_path: /etc/letsencrypt/live/example.com/privkey.pem
+            ytdl_base_path: http://localhost:17442/
+            ytdl_audio_folder_path: audio/
+            ytdl_video_folder_path: video/
+            ytdl_title_top: Youtube Downloader
+            ytdl_file_manager_enabled: 'true'
+            ytdl_allow_quality_select: 'true'
+            ytdl_download_only_mode: 'false'
+            ytdl_use_youtube_api: 'false'
+            ytdl_youtube_api_key: 'false'
+            ytdl_default_theme: default
+            ytdl_allow_theme_change: 'true'
+            ytdl_use_default_downloading_agent: 'true'
+            ytdl_custom_downloading_agent: 'false'
+            write_ytdl_config: 'true'
+            # do not touch this
+            ALLOW_CONFIG_MUTATIONS: 'true'
+        restart: always
+        ports:
+            - "17442:17442"
+            - "8998:80"
+        image: tzahi12345/youtubedl-material

docker_wrapper.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+cd backend
+
+# Start the first process
+node app.js &
+status=$?
+if [ $status -ne 0 ]; then
+  echo "Failed to start my_first_process: $status"
+  exit $status
+fi
+
+# Start the second process
+apachectl -DFOREGROUND
+status=$?
+if [ $status -ne 0 ]; then
+  echo "Failed to start my_second_process: $status"
+  exit $status
+fi
+
+# Naive check runs checks once a minute to see if either of the processes exited.
+# This illustrates part of the heavy lifting you need to do if you want to run
+# more than one service in a container. The container will exit with an error
+# if it detects that either of the processes has exited.
+# Otherwise it will loop forever, waking up every 60 seconds
+
+while /bin/true; do
+  ps aux |grep node\ app.js # |grep -q -v grep
+  PROCESS_1_STATUS=$?
+  ps aux |grep apache2 # |grep -q -v grep
+  PROCESS_2_STATUS=$?
+  # If the greps above find anything, they will exit with 0 status
+  # If they are not both 0, then something is wrong
+  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
+    echo "One of the processes has already exited."
+    exit -1
+  fi
+  sleep 60
+done

package.json

@@ -9,8 +9,7 @@
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e",
-    "electron": "ng build --base-href ./ && electron .",
-    "postinstall": "ng build --prod && mkdir dist/backend && mkdir dist/backend/config && mkdir dist/backend/audio && mkdir dist/backend/video && cp src/assets/default.json dist/backend/config/default.json && cp backend/app.js dist/backend/app.js && cp backend/package.json dist/backend/package.json && cd dist/backend && npm install"
+    "electron": "ng build --base-href ./ && electron ."
   },
   "engines": {
     "node": "12.3.1",