@ -32,6 +32,9 @@ const FileSync = require('lowdb/adapters/FileSync')
const adapter = new FileSync ( './appdata/db.json' ) ;
const db = low ( adapter )
// check if debug mode
let debugMode = process . env . YTDL _MODE === 'debug' ;
// logging setup
// console format
@ -39,7 +42,7 @@ const defaultFormat = winston.format.printf(({ level, message, label, timestamp
return ` ${ timestamp } ${ level . toUpperCase ( ) } : ${ message } ` ;
} ) ;
const logger = winston . createLogger ( {
level : 'info ',
level : ! debugMode ? 'info ' : 'debug ',
format : winston . format . combine ( winston . format . timestamp ( ) , defaultFormat ) ,
defaultMeta : { } ,
transports : [
@ -65,9 +68,14 @@ db.defaults(
audio : [ ] ,
video : [ ]
} ,
files : {
audio : [ ] ,
video : [ ]
} ,
configWriteFlag : false ,
subscriptions : [ ] ,
pin _md5 : ''
pin _md5 : '' ,
files _to _db _migration _complete : false
} ) . write ( ) ;
// config values
@ -90,9 +98,6 @@ var options = null; // encryption options
var url _domain = null ;
var updaterStatus = null ;
// check if debug mode
let debugMode = process . env . YTDL _MODE === 'debug' ;
if ( debugMode ) logger . info ( 'YTDL-Material in debug mode!' ) ;
// check if just updated
@ -154,6 +159,56 @@ function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, p
// actual functions
async function checkMigrations ( ) {
return new Promise ( async resolve => {
// 3.5->3.6 migration
const files _to _db _migration _complete = db . get ( 'files_to_db_migration_complete' ) . value ( ) ;
if ( ! files _to _db _migration _complete ) {
logger . info ( 'Beginning migration: 3.5->3.6+' )
runFilesToDBMigration ( ) . then ( success => {
if ( success ) { logger . info ( '3.5->3.6+ migration complete!' ) ; }
else { logger . error ( 'Migration failed: 3.5->3.6+' ) ; }
} ) ;
}
resolve ( true ) ;
} ) ;
}
async function runFilesToDBMigration ( ) {
return new Promise ( async resolve => {
try {
let mp3s = getMp3s ( ) ;
let mp4s = getMp4s ( ) ;
for ( let i = 0 ; i < mp3s . length ; i ++ ) {
let file _obj = mp3s [ i ] ;
const file _already _in _db = db . get ( 'files.audio' ) . find ( { id : file _obj . id } ) . value ( ) ;
if ( ! file _already _in _db ) {
logger . verbose ( ` Migrating file ${ file _obj . id } ` ) ;
registerFileDB ( file _obj . id + '.mp3' , 'audio' ) ;
}
}
for ( let i = 0 ; i < mp4s . length ; i ++ ) {
let file _obj = mp4s [ i ] ;
const file _already _in _db = db . get ( 'files.video' ) . find ( { id : file _obj . id } ) . value ( ) ;
if ( ! file _already _in _db ) {
logger . verbose ( ` Migrating file ${ file _obj . id } ` ) ;
registerFileDB ( file _obj . id + '.mp4' , 'video' ) ;
}
}
// sets migration to complete
db . set ( 'files_to_db_migration_complete' , true ) . write ( ) ;
resolve ( true ) ;
} catch ( err ) {
resolve ( false ) ;
}
} ) ;
}
async function startServer ( ) {
if ( process . env . USING _HEROKU && process . env . PORT ) {
// default to heroku port if using heroku
@ -435,7 +490,7 @@ async function setConfigFromEnv() {
}
async function loadConfig ( ) {
return new Promise ( resolve => {
return new Promise ( async resolve => {
url = ! debugMode ? config _api . getConfigItem ( 'ytdl_url' ) : 'http://localhost:4200' ;
backendPort = config _api . getConfigItem ( 'ytdl_port' ) ;
usingEncryption = config _api . getConfigItem ( 'ytdl_use_encryption' ) ;
@ -483,6 +538,9 @@ async function loadConfig() {
} , subscriptionsCheckInterval * 1000 ) ;
}
// check migrations
await checkMigrations ( ) ;
// start the server here
startServer ( ) ;
@ -544,6 +602,64 @@ function generateEnvVarConfigItem(key) {
return { key : key , value : process [ 'env' ] [ key ] } ;
}
function getMp3s ( ) {
let mp3s = [ ] ;
var files = recFindByExt ( audioFolderPath , 'mp3' ) ; // fs.readdirSync(audioFolderPath);
for ( let i = 0 ; i < files . length ; i ++ ) {
let file = files [ i ] ;
var file _path = file . substring ( audioFolderPath . length , file . length ) ;
var stats = fs . statSync ( file ) ;
var id = file _path . substring ( 0 , file _path . length - 4 ) ;
var jsonobj = getJSONMp3 ( id ) ;
if ( ! jsonobj ) continue ;
var title = jsonobj . title ;
var url = jsonobj . webpage _url ;
var uploader = jsonobj . uploader ;
var upload _date = jsonobj . upload _date ;
upload _date = ` ${ upload _date . substring ( 0 , 4 ) } - ${ upload _date . substring ( 4 , 6 ) } - ${ upload _date . substring ( 6 , 8 ) } ` ;
var size = stats . size ;
var thumbnail = jsonobj . thumbnail ;
var duration = jsonobj . duration ;
var isaudio = true ;
var file _obj = new File ( id , title , thumbnail , isaudio , duration , url , uploader , size , file , upload _date ) ;
mp3s . push ( file _obj ) ;
}
return mp3s ;
}
function getMp4s ( relative _path = true ) {
let mp4s = [ ] ;
var files = recFindByExt ( videoFolderPath , 'mp4' ) ;
for ( let i = 0 ; i < files . length ; i ++ ) {
let file = files [ i ] ;
var file _path = file . substring ( videoFolderPath . length , file . length ) ;
var stats = fs . statSync ( file ) ;
var id = file _path . substring ( 0 , file _path . length - 4 ) ;
var jsonobj = getJSONMp4 ( id ) ;
if ( ! jsonobj ) continue ;
var title = jsonobj . title ;
var url = jsonobj . webpage _url ;
var uploader = jsonobj . uploader ;
var upload _date = jsonobj . upload _date ;
upload _date = ` ${ upload _date . substring ( 0 , 4 ) } - ${ upload _date . substring ( 4 , 6 ) } - ${ upload _date . substring ( 6 , 8 ) } ` ;
var thumbnail = jsonobj . thumbnail ;
var duration = jsonobj . duration ;
var size = stats . size ;
var isaudio = false ;
var file _obj = new File ( id , title , thumbnail , isaudio , duration , url , uploader , size , file , upload _date ) ;
mp4s . push ( file _obj ) ;
}
return mp4s ;
}
function getThumbnailMp3 ( name )
{
var obj = getJSONMp3 ( name ) ;
@ -856,6 +972,63 @@ function recFindByExt(base,ext,files,result)
return result
}
function registerFileDB ( full _file _path , type ) {
const file _id = full _file _path . substring ( 0 , full _file _path . length - 4 ) ;
const file _object = generateFileObject ( file _id , type ) ;
if ( ! file _object ) {
logger . error ( ` Could not find associated JSON file for ${ type } file ${ file _id } ` ) ;
return false ;
}
file _object [ 'uid' ] = uuid ( ) ;
path _object = path . parse ( file _object [ 'path' ] ) ;
file _object [ 'path' ] = path . format ( path _object ) ;
db . get ( ` files. ${ type } ` )
. push ( file _object )
. write ( ) ;
return file _object [ 'uid' ] ;
}
function generateFileObject ( id , type ) {
var jsonobj = ( type === 'audio' ) ? getJSONMp3 ( id ) : getJSONMp4 ( id ) ;
if ( ! jsonobj ) {
return null ;
}
const ext = ( type === 'audio' ) ? '.mp3' : '.mp4'
const file _path = getTrueFileName ( jsonobj [ '_filename' ] , type ) ; // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
var stats = fs . statSync ( path . join ( _ _dirname , file _path ) ) ;
var title = jsonobj . title ;
var url = jsonobj . webpage _url ;
var uploader = jsonobj . uploader ;
var upload _date = jsonobj . upload _date ;
upload _date = ` ${ upload _date . substring ( 0 , 4 ) } - ${ upload _date . substring ( 4 , 6 ) } - ${ upload _date . substring ( 6 , 8 ) } ` ;
var size = stats . size ;
var thumbnail = jsonobj . thumbnail ;
var duration = jsonobj . duration ;
var isaudio = type === 'audio' ;
var file _obj = new File ( id , title , thumbnail , isaudio , duration , url , uploader , size , file _path , upload _date ) ;
return file _obj ;
}
// replaces .webm with appropriate extension
function getTrueFileName ( unfixed _path , type ) {
let fixed _path = unfixed _path ;
const new _ext = ( type === 'audio' ? 'mp3' : 'mp4' ) ;
let unfixed _parts = unfixed _path . split ( '.' ) ;
const old _ext = unfixed _parts [ unfixed _parts . length - 1 ] ;
if ( old _ext !== new _ext ) {
unfixed _parts [ unfixed _parts . length - 1 ] = new _ext ;
fixed _path = unfixed _parts . join ( '.' ) ;
}
return fixed _path ;
}
function getAudioInfos ( fileNames ) {
let result = [ ] ;
for ( let i = 0 ; i < fileNames . length ; i ++ ) {
@ -1156,6 +1329,7 @@ app.post('/api/tomp3', async function(req, res) {
}
youtubedl . exec ( url , downloadConfig , { } , function ( err , output ) {
var uid = null ;
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` Audio download delay: ${ difference } seconds. ` ) ;
@ -1182,9 +1356,10 @@ app.post('/api/tomp3', async function(req, res) {
continue ;
}
const file name _no _extension = removeFileExtension ( output _json [ '_filename' ] ) ;
const file path _no _extension = removeFileExtension ( output _json [ '_filename' ] ) ;
var full _file _path = filename _no _extension + '.mp3' ;
var full _file _path = filepath _no _extension + '.mp3' ;
var file _name = filepath _no _extension . substring ( audioFolderPath . length , filepath _no _extension . length ) ;
if ( fs . existsSync ( full _file _path ) ) {
let tags = {
title : output _json [ 'title' ] ,
@ -1193,12 +1368,14 @@ app.post('/api/tomp3', async function(req, res) {
// NodeID3.create(tags, function(frame) { })
let success = NodeID3 . write ( tags , full _file _path ) ;
if ( ! success ) logger . error ( 'Failed to apply ID3 tag to audio file ' + full _file _path ) ;
// registers file in DB
uid = registerFileDB ( full _file _path . substring ( audioFolderPath . length , full _file _path . length ) , 'audio' ) ;
} else {
logger . info ( 'Output mp3 does not exist' ) ;
logger . error( 'Download failed: Output mp3 does not exist') ;
}
var file _path = filename _no _extension . substring ( audioFolderPath . length , filename _no _extension . length ) ;
if ( file _path ) file _names . push ( file _path ) ;
if ( file _name ) file _names . push ( file _name ) ;
}
let is _playlist = file _names . length > 1 ;
@ -1214,7 +1391,8 @@ app.post('/api/tomp3', async function(req, res) {
var audiopathEncoded = encodeURIComponent ( file _names [ 0 ] ) ;
res . send ( {
audiopathEncoded : audiopathEncoded ,
file _names : is _playlist ? file _names : null
file _names : is _playlist ? file _names : null ,
uid : uid
} ) ;
}
} ) ;
@ -1293,6 +1471,7 @@ app.post('/api/tomp4', async function(req, res) {
}
youtubedl . exec ( url , downloadConfig , { } , function ( err , output ) {
var uid = null ;
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` Video download delay: ${ difference } seconds. ` ) ;
@ -1318,7 +1497,12 @@ app.post('/api/tomp4', async function(req, res) {
if ( ! output _json ) {
continue ;
}
var file _name = output _json [ '_filename' ] . replace ( /^.*[\\\/]/ , '' ) ;
// get filepath with no extension
const filepath _no _extension = removeFileExtension ( output _json [ '_filename' ] ) ;
var full _file _path = filepath _no _extension + '.mp4' ;
var file _name = filepath _no _extension . substring ( audioFolderPath . length , filepath _no _extension . length ) ;
// renames file if necessary due to bug
if ( ! fs . existsSync ( output _json [ '_filename' ] && fs . existsSync ( output _json [ '_filename' ] + '.webm' ) ) ) {
@ -1328,11 +1512,11 @@ app.post('/api/tomp4', async function(req, res) {
} catch ( e ) {
}
}
var alternate _file _name = file _name . substring ( 0 , file _name . length - 4 ) ;
var file _path = output _json [ '_filename' ] . substring ( audioFolderPath . length , output _json [ '_filename' ] . length ) ;
// remove extension from file path
var alternate _file _path = file _path . replace ( /\.[^/.]+$/ , "" )
if ( alternate_ file_name ) file _names . push ( alternate_file _path ) ;
// registers file in DB
uid = registerFileDB ( full _file _path . substring ( videoFolderPath . length , full _file _path . length ) , 'video' ) ;
if ( file_name ) file _names . push ( file_name ) ;
}
let is _playlist = file _names . length > 1 ;
@ -1348,7 +1532,8 @@ app.post('/api/tomp4', async function(req, res) {
var videopathEncoded = encodeURIComponent ( file _names [ 0 ] ) ;
res . send ( {
videopathEncoded : videopathEncoded ,
file _names : is _playlist ? file _names : null
file _names : is _playlist ? file _names : null ,
uid : uid
} ) ;
res . end ( "yes" ) ;
}
@ -1399,32 +1584,8 @@ app.post('/api/fileStatusMp4', function(req, res) {
// gets all download mp3s
app . post ( '/api/getMp3s' , function ( req , res ) {
var mp3s = [ ] ;
var mp3s = db . get ( 'files.audio' ) . value ( ) ; // getMp3s();
var playlists = db . get ( 'playlists.audio' ) . value ( ) ;
var files = recFindByExt ( audioFolderPath , 'mp3' ) ; // fs.readdirSync(audioFolderPath);
for ( let i = 0 ; i < files . length ; i ++ ) {
let file = files [ i ] ;
var file _path = file . substring ( audioFolderPath . length , file . length ) ;
var stats = fs . statSync ( file ) ;
var id = file _path . substring ( 0 , file _path . length - 4 ) ;
var jsonobj = getJSONMp3 ( id ) ;
if ( ! jsonobj ) continue ;
var title = jsonobj . title ;
var url = jsonobj . webpage _url ;
var uploader = jsonobj . uploader ;
var upload _date = jsonobj . upload _date ;
upload _date = ` ${ upload _date . substring ( 0 , 4 ) } - ${ upload _date . substring ( 4 , 6 ) } - ${ upload _date . substring ( 6 , 8 ) } ` ;
var size = stats . size ;
var thumbnail = jsonobj . thumbnail ;
var duration = jsonobj . duration ;
var isaudio = true ;
var file _obj = new File ( id , title , thumbnail , isaudio , duration , url , uploader , size , file , upload _date ) ;
mp3s . push ( file _obj ) ;
}
res . send ( {
mp3s : mp3s ,
@ -1435,39 +1596,111 @@ app.post('/api/getMp3s', function(req, res) {
// gets all download mp4s
app . post ( '/api/getMp4s' , function ( req , res ) {
var mp4s = [ ] ;
var mp4s = db . get ( 'files.video' ) . value ( ) ; // getMp4s();
var playlists = db . get ( 'playlists.video' ) . value ( ) ;
var fullpath = videoFolderPath ;
var files = recFindByExt ( videoFolderPath , 'mp4' ) ;
for ( let i = 0 ; i < files . length ; i ++ ) {
let file = files [ i ] ;
var file _path = file . substring ( videoFolderPath . length , file . length ) ;
var stats = fs . statSync ( file ) ;
res . send ( {
mp4s : mp4s ,
playlists : playlists
} ) ;
res . end ( "yes" ) ;
} ) ;
app . post ( '/api/getFile' , function ( req , res ) {
var uid = req . body . uid ;
var type = req . body . type ;
var file = null ;
if ( ! type ) {
file = db . get ( 'files.audio' ) . find ( { uid : uid } ) . value ( ) ;
if ( ! file ) {
file = db . get ( 'files.video' ) . find ( { uid : uid } ) . value ( ) ;
if ( file ) type = 'video' ;
} else {
type = 'audio' ;
}
}
if ( ! file && type ) db . get ( ` files. ${ type } ` ) . find ( { uid : uid } ) . value ( ) ;
if ( file ) {
res . send ( {
success : true ,
file : file
} ) ;
} else {
res . send ( {
success : false
} ) ;
}
} ) ;
// video sharing
app . post ( '/api/enableSharing' , function ( req , res ) {
var type = req . body . type ;
var uid = req . body . uid ;
var is _playlist = req . body . is _playlist ;
try {
success = true ;
if ( ! is _playlist && type !== 'subscription' ) {
db . get ( ` files. ${ type } ` )
. find ( { uid : uid } )
. assign ( { sharingEnabled : true } )
. write ( ) ;
} else if ( is _playlist ) {
db . get ( ` playlists. ${ type } ` )
. find ( { id : uid } )
. assign ( { sharingEnabled : true } )
. write ( ) ;
} else if ( type === 'subscription' ) {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
} else {
// error
success = false ;
}
var id = file _path . substring ( 0 , file _path . length - 4 ) ;
var jsonobj = getJSONMp4 ( id ) ;
if ( ! jsonobj ) continue ;
var title = jsonobj . title ;
var url = jsonobj . webpage _url ;
var uploader = jsonobj . uploader ;
var upload _date = jsonobj . upload _date ;
upload _date = ` ${ upload _date . substring ( 0 , 4 ) } - ${ upload _date . substring ( 4 , 6 ) } - ${ upload _date . substring ( 6 , 8 ) } ` ;
var thumbnail = jsonobj . thumbnail ;
var duration = jsonobj . duration ;
} catch ( err ) {
success = false ;
}
var size = stats . size ;
res . send ( {
success : success
} ) ;
} ) ;
var isaudio = false ;
var file _obj = new File ( id , title , thumbnail , isaudio , duration , url , uploader , size , file , upload _date ) ;
mp4s . push ( file _obj ) ;
app . post ( '/api/disableSharing' , function ( req , res ) {
var type = req . body . type ;
var uid = req . body . uid ;
var is _playlist = req . body . is _playlist ;
try {
success = true ;
if ( ! is _playlist && type !== 'subscription' ) {
db . get ( ` files. ${ type } ` )
. find ( { uid : uid } )
. assign ( { sharingEnabled : false } )
. write ( ) ;
} else if ( is _playlist ) {
db . get ( ` playlists. ${ type } ` )
. find ( { id : uid } )
. assign ( { sharingEnabled : false } )
. write ( ) ;
} else if ( type === 'subscription' ) {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
} else {
// error
success = false ;
}
} catch ( err ) {
success = false ;
}
res . send ( {
mp4s : mp4s ,
playlists : playlists
success : success
} ) ;
res . end ( "yes" ) ;
} ) ;
app . post ( '/api/subscribe' , async ( req , res ) => {
@ -1620,7 +1853,8 @@ app.post('/api/createPlaylist', async (req, res) => {
'name' : playlistName ,
fileNames : fileNames ,
id : shortid . generate ( ) ,
thumbnailURL : thumbnailURL
thumbnailURL : thumbnailURL ,
type : type
} ;
db . get ( ` playlists. ${ type } ` )
@ -1633,6 +1867,31 @@ app.post('/api/createPlaylist', async (req, res) => {
} )
} ) ;
app . post ( '/api/getPlaylist' , async ( req , res ) => {
let playlistID = req . body . playlistID ;
let type = req . body . type ;
let playlist = null ;
if ( ! type ) {
playlist = db . get ( 'playlists.audio' ) . find ( { id : playlistID } ) . value ( ) ;
if ( ! playlist ) {
playlist = db . get ( 'playlists.video' ) . find ( { id : playlistID } ) . value ( ) ;
if ( playlist ) type = 'video' ;
} else {
type = 'audio' ;
}
}
if ( ! playlist ) playlist = db . get ( ` playlists. ${ type } ` ) . find ( { id : playlistID } ) . value ( ) ;
res . send ( {
playlist : playlist ,
type : type ,
success : ! ! playlist
} ) ;
} ) ;
app . post ( '/api/updatePlaylist' , async ( req , res ) => {
let playlistID = req . body . playlistID ;
let fileNames = req . body . fileNames ;
@ -1682,40 +1941,50 @@ app.post('/api/deletePlaylist', async (req, res) => {
// deletes mp3 file
app . post ( '/api/deleteMp3' , async ( req , res ) => {
var name = req . body . name ;
// var name = req.body.name;
var uid = req . body . uid ;
var audio _obj = db . get ( 'files.audio' ) . find ( { uid : uid } ) . value ( ) ;
var name = audio _obj . id ;
var blacklistMode = req . body . blacklistMode ;
var fullpath = audioFolderPath + name + ".mp3" ;
var wasDeleted = false ;
if ( fs . existsSync ( fullpath ) )
{
deleteAudioFile ( name , blacklistMode ) ;
db . get ( 'files.audio' ) . remove ( { uid : uid } ) . write ( ) ;
wasDeleted = true ;
res . send ( wasDeleted ) ;
res . end ( "yes" ) ;
}
else
{
} else if ( audio _obj ) {
db . get ( 'files.audio' ) . remove ( { uid : uid } ) . write ( ) ;
wasDeleted = true ;
res . send ( wasDeleted ) ;
} else {
wasDeleted = false ;
res . send ( wasDeleted ) ;
res . end ( "yes" ) ;
}
} ) ;
// deletes mp4 file
app . post ( '/api/deleteMp4' , async ( req , res ) => {
var name = req . body . name ;
var uid = req . body . uid ;
var video _obj = db . get ( 'files.video' ) . find ( { uid : uid } ) . value ( ) ;
var name = video _obj . id ;
var blacklistMode = req . body . blacklistMode ;
var fullpath = videoFolderPath + name + ".mp4" ;
var wasDeleted = false ;
if ( fs . existsSync ( fullpath ) )
{
wasDeleted = await deleteVideoFile ( name , null , blacklistMode ) ;
db . get ( 'files.video' ) . remove ( { uid : uid } ) . write ( ) ;
// wasDeleted = true;
res . send ( wasDeleted ) ;
res . end ( "yes" ) ;
}
else
{
} else if ( video _obj ) {
db . get ( 'files.video' ) . remove ( { uid : uid } ) . write ( ) ;
wasDeleted = true ;
res . send ( wasDeleted ) ;
} else {
wasDeleted = false ;
res . send ( wasDeleted ) ;
res . end ( "yes" ) ;