Merge branch 'create-playlist-cleaner'

package.json

@@ -12,6 +12,7 @@
     "playlist:generate": "node scripts/commands/playlist/generate.js",
     "playlist:update": "node scripts/commands/playlist/update.js",
     "playlist:lint": "npx m3u-linter -c m3u-linter.json",
+    "playlist:cleaner": "node scripts/commands/playlist/cleaner.js",
     "readme:update": "node scripts/commands/readme/update.js",
     "test": "jest --runInBand"
   },

scripts/commands/database/create.js

@@ -33,8 +33,8 @@ async function findStreams() {
   const streams = []
   const files = await file.list(`${options.inputDir}/**/*.m3u`)
   for (const filepath of files) {
-    const items = await parser.parsePlaylist(filepath)
-    for (const item of items) {
+    const playlist = await parser.parsePlaylist(filepath)
+    for (const item of playlist.items) {
       item.filepath = filepath
 
       const stream = store.create()

scripts/commands/playlist/cleaner.js

@@ -0,0 +1,90 @@
+const { file, parser, logger, checker, m3u } = require('../../core')
+const { program } = require('commander')
+
+program
+  .argument('[filepath]', 'Path to file to validate')
+  .option('-t, --timeout <timeout>', 'Set timeout for each request', parser.parseNumber, 60000)
+  .option('-d, --delay <delay>', 'Set delay for each request', parser.parseNumber, 0)
+  .option('--debug', 'Enable debug mode')
+  .parse(process.argv)
+
+const options = program.opts()
+
+async function main() {
+  const files = program.args.length ? program.args : await file.list('streams/*.m3u')
+
+  for (const filepath of files) {
+    if (!filepath.endsWith('.m3u')) continue
+    logger.info(`${filepath}`)
+    const playlist = await parser.parsePlaylist(filepath)
+    const before = playlist.items.length
+    for (const stream of playlist.items) {
+      if (options.debug) logger.info(stream.url)
+      const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null]
+      stream.status = status
+      if (status === 'error' && /^(http|https)/.test(stream.url) && !/\[.*\]$/.test(stream.name)) {
+        const result = await checkStream(stream)
+        const newStatus = parseStatus(result.error)
+        if (status === newStatus) {
+          stream.remove = true
+          logger.info(`removed "${stream.name}"`)
+        }
+      }
+    }
+
+    const items = playlist.items
+      .filter(i => !i.remove)
+      .map(item => ({
+        attrs: {
+          'tvg-id': item.tvg.id,
+          status: item.status,
+          'user-agent': item.http['user-agent'] || undefined
+        },
+        title: item.name,
+        url: item.url,
+        vlcOpts: {
+          'http-referrer': item.http.referrer || undefined,
+          'http-user-agent': item.http['user-agent'] || undefined
+        }
+      }))
+
+    if (before !== items.length) {
+      const output = m3u.create(items)
+      await file.create(filepath, output)
+      logger.info(`saved`)
+    }
+  }
+}
+
+main()
+
+async function checkStream(item) {
+  const config = {
+    timeout: options.timeout,
+    delay: options.delay,
+    debug: options.debug
+  }
+
+  const request = {
+    url: item.url,
+    http: {
+      referrer: item.http.referrer,
+      'user-agent': item.http['user-agent']
+    }
+  }
+
+  return checker.check(request, config)
+}
+
+function parseStatus(error) {
+  if (!error) return 'online'
+
+  switch (error) {
+    case 'Operation timed out':
+      return 'timeout'
+    case 'Server returned 403 Forbidden (access denied)':
+      return 'blocked'
+    default:
+      return 'error'
+  }
+}

scripts/commands/playlist/validate.js

@@ -6,7 +6,7 @@ const _ = require('lodash')
 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
 
 async function main() {
-  const files = program.args.length ? program.args : await file.list('channels/*.m3u')
+  const files = program.args.length ? program.args : await file.list('streams/*.m3u')
 
   logger.info(`loading blocklist...`)
   await api.channels.load()
@@ -31,8 +31,8 @@ async function main() {
     const [__, country] = basename.match(/([a-z]{2})(|_.*)\.m3u/i) || [null, null]
 
     const fileLog = []
-    const items = await parser.parsePlaylist(filepath)
-    for (const item of items) {
+    const playlist = await parser.parsePlaylist(filepath)
+    for (const item of playlist.items) {
       if (item.tvg.id && !api.channels.find({ id: item.tvg.id })) {
         fileLog.push({
           type: 'warning',

scripts/core/index.js

@@ -10,3 +10,4 @@ exports.store = require('./store')
 exports.markdown = require('./markdown')
 exports.api = require('./api')
 exports.id = require('./id')
+exports.m3u = require('./m3u')

scripts/core/m3u.js

@@ -0,0 +1,34 @@
+const m3u = {}
+
+m3u.create = function (links = [], header = {}) {
+	let output = `#EXTM3U`
+	for (const attr in header) {
+		const value = header[attr]
+		output += ` ${attr}="${value}"`
+	}
+	output += `\n`
+
+	for (const link of links) {
+		output += `#EXTINF:-1`
+		for (const name in link.attrs) {
+			const value = link.attrs[name]
+			if (value !== undefined) {
+				output += ` ${name}="${value}"`
+			}
+		}
+		output += `,${link.title}\n`
+
+		for (const name in link.vlcOpts) {
+			const value = link.vlcOpts[name]
+			if (value !== undefined) {
+				output += `#EXTVLCOPT:${name}=${value}\n`
+			}
+		}
+
+		output += `${link.url}\n`
+	}
+
+	return output
+}
+
+module.exports = m3u

scripts/core/parser.js

@@ -6,9 +6,8 @@ const parser = {}
 
 parser.parsePlaylist = async function (filepath) {
   const content = await file.read(filepath)
-  const playlist = ipp.parse(content)
 
-  return playlist.items
+  return ipp.parse(content)
 }
 
 parser.parseLogs = async function (filepath) {

scripts/core/playlist.js

@@ -1,4 +1,5 @@
 const store = require('./store')
+const m3u = require('./m3u')
 const _ = require('lodash')
 
 const playlist = {}
@@ -50,34 +51,7 @@ class Playlist {
   }
 
   toString() {
-    let output = `#EXTM3U`
-    for (const attr in this.header) {
-      const value = this.header[attr]
-      output += ` ${attr}="${value}"`
-    }
-    output += `\n`
-
-    for (const link of this.links) {
-      output += `#EXTINF:-1`
-      for (const name in link.attrs) {
-        const value = link.attrs[name]
-        if (value !== undefined) {
-          output += ` ${name}="${value}"`
-        }
-      }
-      output += `,${link.title}\n`
-
-      for (const name in link.vlcOpts) {
-        const value = link.vlcOpts[name]
-        if (value !== undefined) {
-          output += `#EXTVLCOPT:${name}=${value}\n`
-        }
-      }
-
-      output += `${link.url}\n`
-    }
-
-    return output
+    return m3u.create(this.links, this.header)
   }
 }
 

tests/__data__/expected/database/db_create.streams.db

@@ -1,4 +1,6 @@
-{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"ZJejfvbOVTyuf6Gk"}
-{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"gnjGLZU1CEz79gcp"}
-{"channel":"ATV.ad","title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"9r9qmYRa2kxiirl0"}
-{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"unOCFJtsDCbJupxR"}
+{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"}
+{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"}
+{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"}
+{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"}
+{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"}
+{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"}