diff --git a/package.json b/package.json index 56673d549..4e489ee78 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "playlist:generate": "node scripts/commands/playlist/generate.js", "playlist:update": "node scripts/commands/playlist/update.js", "playlist:lint": "npx m3u-linter -c m3u-linter.json", + "playlist:cleaner": "node scripts/commands/playlist/cleaner.js", "readme:update": "node scripts/commands/readme/update.js", "test": "jest --runInBand" }, diff --git a/scripts/commands/database/create.js b/scripts/commands/database/create.js index ec9e9c558..d4f54666b 100644 --- a/scripts/commands/database/create.js +++ b/scripts/commands/database/create.js @@ -33,8 +33,8 @@ async function findStreams() { const streams = [] const files = await file.list(`${options.inputDir}/**/*.m3u`) for (const filepath of files) { - const items = await parser.parsePlaylist(filepath) - for (const item of items) { + const playlist = await parser.parsePlaylist(filepath) + for (const item of playlist.items) { item.filepath = filepath const stream = store.create() diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js new file mode 100644 index 000000000..8595dbf5f --- /dev/null +++ b/scripts/commands/playlist/cleaner.js @@ -0,0 +1,90 @@ +const { file, parser, logger, checker, m3u } = require('../../core') +const { program } = require('commander') + +program + .argument('[filepath]', 'Path to file to validate') + .option('-t, --timeout ', 'Set timeout for each request', parser.parseNumber, 60000) + .option('-d, --delay ', 'Set delay for each request', parser.parseNumber, 0) + .option('--debug', 'Enable debug mode') + .parse(process.argv) + +const options = program.opts() + +async function main() { + const files = program.args.length ? program.args : await file.list('streams/*.m3u') + + for (const filepath of files) { + if (!filepath.endsWith('.m3u')) continue + logger.info(`${filepath}`) + const playlist = await parser.parsePlaylist(filepath) + const before = playlist.items.length + for (const stream of playlist.items) { + if (options.debug) logger.info(stream.url) + const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] + stream.status = status + if (status === 'error' && /^(http|https)/.test(stream.url) && !/\[.*\]$/.test(stream.name)) { + const result = await checkStream(stream) + const newStatus = parseStatus(result.error) + if (status === newStatus) { + stream.remove = true + logger.info(`removed "${stream.name}"`) + } + } + } + + const items = playlist.items + .filter(i => !i.remove) + .map(item => ({ + attrs: { + 'tvg-id': item.tvg.id, + status: item.status, + 'user-agent': item.http['user-agent'] || undefined + }, + title: item.name, + url: item.url, + vlcOpts: { + 'http-referrer': item.http.referrer || undefined, + 'http-user-agent': item.http['user-agent'] || undefined + } + })) + + if (before !== items.length) { + const output = m3u.create(items) + await file.create(filepath, output) + logger.info(`saved`) + } + } +} + +main() + +async function checkStream(item) { + const config = { + timeout: options.timeout, + delay: options.delay, + debug: options.debug + } + + const request = { + url: item.url, + http: { + referrer: item.http.referrer, + 'user-agent': item.http['user-agent'] + } + } + + return checker.check(request, config) +} + +function parseStatus(error) { + if (!error) return 'online' + + switch (error) { + case 'Operation timed out': + return 'timeout' + case 'Server returned 403 Forbidden (access denied)': + return 'blocked' + default: + return 'error' + } +} diff --git a/scripts/commands/playlist/validate.js b/scripts/commands/playlist/validate.js index 2a06eff95..2b1b4badb 100644 --- a/scripts/commands/playlist/validate.js +++ b/scripts/commands/playlist/validate.js @@ -6,7 +6,7 @@ const _ = require('lodash') program.argument('[filepath]', 'Path to file to validate').parse(process.argv) async function main() { - const files = program.args.length ? program.args : await file.list('channels/*.m3u') + const files = program.args.length ? program.args : await file.list('streams/*.m3u') logger.info(`loading blocklist...`) await api.channels.load() @@ -31,8 +31,8 @@ async function main() { const [__, country] = basename.match(/([a-z]{2})(|_.*)\.m3u/i) || [null, null] const fileLog = [] - const items = await parser.parsePlaylist(filepath) - for (const item of items) { + const playlist = await parser.parsePlaylist(filepath) + for (const item of playlist.items) { if (item.tvg.id && !api.channels.find({ id: item.tvg.id })) { fileLog.push({ type: 'warning', diff --git a/scripts/core/index.js b/scripts/core/index.js index e8e88ffd0..d5a5b727c 100644 --- a/scripts/core/index.js +++ b/scripts/core/index.js @@ -10,3 +10,4 @@ exports.store = require('./store') exports.markdown = require('./markdown') exports.api = require('./api') exports.id = require('./id') +exports.m3u = require('./m3u') diff --git a/scripts/core/m3u.js b/scripts/core/m3u.js new file mode 100644 index 000000000..5b93f22ed --- /dev/null +++ b/scripts/core/m3u.js @@ -0,0 +1,34 @@ +const m3u = {} + +m3u.create = function (links = [], header = {}) { + let output = `#EXTM3U` + for (const attr in header) { + const value = header[attr] + output += ` ${attr}="${value}"` + } + output += `\n` + + for (const link of links) { + output += `#EXTINF:-1` + for (const name in link.attrs) { + const value = link.attrs[name] + if (value !== undefined) { + output += ` ${name}="${value}"` + } + } + output += `,${link.title}\n` + + for (const name in link.vlcOpts) { + const value = link.vlcOpts[name] + if (value !== undefined) { + output += `#EXTVLCOPT:${name}=${value}\n` + } + } + + output += `${link.url}\n` + } + + return output +} + +module.exports = m3u diff --git a/scripts/core/parser.js b/scripts/core/parser.js index b54fd8f5a..ac0e31a72 100644 --- a/scripts/core/parser.js +++ b/scripts/core/parser.js @@ -6,9 +6,8 @@ const parser = {} parser.parsePlaylist = async function (filepath) { const content = await file.read(filepath) - const playlist = ipp.parse(content) - return playlist.items + return ipp.parse(content) } parser.parseLogs = async function (filepath) { diff --git a/scripts/core/playlist.js b/scripts/core/playlist.js index fdf57aec9..eefc0a494 100644 --- a/scripts/core/playlist.js +++ b/scripts/core/playlist.js @@ -1,4 +1,5 @@ const store = require('./store') +const m3u = require('./m3u') const _ = require('lodash') const playlist = {} @@ -50,34 +51,7 @@ class Playlist { } toString() { - let output = `#EXTM3U` - for (const attr in this.header) { - const value = this.header[attr] - output += ` ${attr}="${value}"` - } - output += `\n` - - for (const link of this.links) { - output += `#EXTINF:-1` - for (const name in link.attrs) { - const value = link.attrs[name] - if (value !== undefined) { - output += ` ${name}="${value}"` - } - } - output += `,${link.title}\n` - - for (const name in link.vlcOpts) { - const value = link.vlcOpts[name] - if (value !== undefined) { - output += `#EXTVLCOPT:${name}=${value}\n` - } - } - - output += `${link.url}\n` - } - - return output + return m3u.create(this.links, this.header) } } diff --git a/tests/__data__/expected/database/db_create.streams.db b/tests/__data__/expected/database/db_create.streams.db index 03306c844..454e77577 100644 --- a/tests/__data__/expected/database/db_create.streams.db +++ b/tests/__data__/expected/database/db_create.streams.db @@ -1,4 +1,6 @@ -{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"ZJejfvbOVTyuf6Gk"} -{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"gnjGLZU1CEz79gcp"} -{"channel":"ATV.ad","title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"9r9qmYRa2kxiirl0"} -{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"unOCFJtsDCbJupxR"} +{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"} +{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"} +{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"} +{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"} +{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"} +{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"}