From 3e369566f004ed52e7de284a932f39e564d2b85f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 17:36:58 +0300 Subject: [PATCH 01/12] Update validate.js --- scripts/commands/playlist/validate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands/playlist/validate.js b/scripts/commands/playlist/validate.js index 2a06eff95..663e73e02 100644 --- a/scripts/commands/playlist/validate.js +++ b/scripts/commands/playlist/validate.js @@ -6,7 +6,7 @@ const _ = require('lodash') program.argument('[filepath]', 'Path to file to validate').parse(process.argv) async function main() { - const files = program.args.length ? program.args : await file.list('channels/*.m3u') + const files = program.args.length ? program.args : await file.list('streams/*.m3u') logger.info(`loading blocklist...`) await api.channels.load() From 70207240d58cd671cf928b9c6fd28dce889e3471 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:01:15 +0300 Subject: [PATCH 02/12] Update package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 56673d549..4e489ee78 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "playlist:generate": "node scripts/commands/playlist/generate.js", "playlist:update": "node scripts/commands/playlist/update.js", "playlist:lint": "npx m3u-linter -c m3u-linter.json", + "playlist:cleaner": "node scripts/commands/playlist/cleaner.js", "readme:update": "node scripts/commands/readme/update.js", "test": "jest --runInBand" }, From c1a6dd4ba1aff95d9b9eb34cda85b0c1103a54ab Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:01:45 +0300 Subject: [PATCH 03/12] Update parsePlaylist() function --- scripts/commands/database/create.js | 4 ++-- scripts/commands/playlist/validate.js | 4 ++-- scripts/core/parser.js | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/commands/database/create.js b/scripts/commands/database/create.js index ec9e9c558..d4f54666b 100644 --- a/scripts/commands/database/create.js +++ b/scripts/commands/database/create.js @@ -33,8 +33,8 @@ async function findStreams() { const streams = [] const files = await file.list(`${options.inputDir}/**/*.m3u`) for (const filepath of files) { - const items = await parser.parsePlaylist(filepath) - for (const item of items) { + const playlist = await parser.parsePlaylist(filepath) + for (const item of playlist.items) { item.filepath = filepath const stream = store.create() diff --git a/scripts/commands/playlist/validate.js b/scripts/commands/playlist/validate.js index 663e73e02..2b1b4badb 100644 --- a/scripts/commands/playlist/validate.js +++ b/scripts/commands/playlist/validate.js @@ -31,8 +31,8 @@ async function main() { const [__, country] = basename.match(/([a-z]{2})(|_.*)\.m3u/i) || [null, null] const fileLog = [] - const items = await parser.parsePlaylist(filepath) - for (const item of items) { + const playlist = await parser.parsePlaylist(filepath) + for (const item of playlist.items) { if (item.tvg.id && !api.channels.find({ id: item.tvg.id })) { fileLog.push({ type: 'warning', diff --git a/scripts/core/parser.js b/scripts/core/parser.js index b54fd8f5a..ac0e31a72 100644 --- a/scripts/core/parser.js +++ b/scripts/core/parser.js @@ -6,9 +6,8 @@ const parser = {} parser.parsePlaylist = async function (filepath) { const content = await file.read(filepath) - const playlist = ipp.parse(content) - return playlist.items + return ipp.parse(content) } parser.parseLogs = async function (filepath) { From 2420f7ba6b74833191cf30ab8f3d51231145998f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:04:58 +0300 Subject: [PATCH 04/12] Update db_create.streams.db --- tests/__data__/expected/database/db_create.streams.db | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/__data__/expected/database/db_create.streams.db b/tests/__data__/expected/database/db_create.streams.db index 03306c844..454e77577 100644 --- a/tests/__data__/expected/database/db_create.streams.db +++ b/tests/__data__/expected/database/db_create.streams.db @@ -1,4 +1,6 @@ -{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"ZJejfvbOVTyuf6Gk"} -{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"gnjGLZU1CEz79gcp"} -{"channel":"ATV.ad","title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"9r9qmYRa2kxiirl0"} -{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"unOCFJtsDCbJupxR"} +{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"} +{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"} +{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"} +{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"} +{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"} +{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"} From 78de33c8769ccbfe1a1017d8dab80d23d29d4c29 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:30:58 +0300 Subject: [PATCH 05/12] Create m3u.js --- scripts/core/m3u.js | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 scripts/core/m3u.js diff --git a/scripts/core/m3u.js b/scripts/core/m3u.js new file mode 100644 index 000000000..5b93f22ed --- /dev/null +++ b/scripts/core/m3u.js @@ -0,0 +1,34 @@ +const m3u = {} + +m3u.create = function (links = [], header = {}) { + let output = `#EXTM3U` + for (const attr in header) { + const value = header[attr] + output += ` ${attr}="${value}"` + } + output += `\n` + + for (const link of links) { + output += `#EXTINF:-1` + for (const name in link.attrs) { + const value = link.attrs[name] + if (value !== undefined) { + output += ` ${name}="${value}"` + } + } + output += `,${link.title}\n` + + for (const name in link.vlcOpts) { + const value = link.vlcOpts[name] + if (value !== undefined) { + output += `#EXTVLCOPT:${name}=${value}\n` + } + } + + output += `${link.url}\n` + } + + return output +} + +module.exports = m3u From 714f38d9d74d742387cd54a59fa448209afb98d0 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:31:00 +0300 Subject: [PATCH 06/12] Update index.js --- scripts/core/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/core/index.js b/scripts/core/index.js index e8e88ffd0..d5a5b727c 100644 --- a/scripts/core/index.js +++ b/scripts/core/index.js @@ -10,3 +10,4 @@ exports.store = require('./store') exports.markdown = require('./markdown') exports.api = require('./api') exports.id = require('./id') +exports.m3u = require('./m3u') From 52921aeffa03f1647cd2adc498845ca1ff7d117d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:31:05 +0300 Subject: [PATCH 07/12] Update playlist.js --- scripts/core/playlist.js | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/scripts/core/playlist.js b/scripts/core/playlist.js index fdf57aec9..eefc0a494 100644 --- a/scripts/core/playlist.js +++ b/scripts/core/playlist.js @@ -1,4 +1,5 @@ const store = require('./store') +const m3u = require('./m3u') const _ = require('lodash') const playlist = {} @@ -50,34 +51,7 @@ class Playlist { } toString() { - let output = `#EXTM3U` - for (const attr in this.header) { - const value = this.header[attr] - output += ` ${attr}="${value}"` - } - output += `\n` - - for (const link of this.links) { - output += `#EXTINF:-1` - for (const name in link.attrs) { - const value = link.attrs[name] - if (value !== undefined) { - output += ` ${name}="${value}"` - } - } - output += `,${link.title}\n` - - for (const name in link.vlcOpts) { - const value = link.vlcOpts[name] - if (value !== undefined) { - output += `#EXTVLCOPT:${name}=${value}\n` - } - } - - output += `${link.url}\n` - } - - return output + return m3u.create(this.links, this.header) } } From 31bc995a3e8f0e9813db8af67a3d73fcfd1357d7 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:32:53 +0300 Subject: [PATCH 08/12] Create cleaner.js --- scripts/commands/playlist/cleaner.js | 81 ++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 scripts/commands/playlist/cleaner.js diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js new file mode 100644 index 000000000..2de731252 --- /dev/null +++ b/scripts/commands/playlist/cleaner.js @@ -0,0 +1,81 @@ +const { file, parser, logger, checker, m3u } = require('../../core') +const { program } = require('commander') + +const options = program + .argument('[filepath]', 'Path to file to validate') + .option('-t, --timeout ', 'Set timeout for each request', parser.parseNumber, 60000) + .option('-d, --delay ', 'Set delay for each request', parser.parseNumber, 0) + .option('--debug', 'Enable debug mode') + .parse(process.argv) + +async function main() { + const files = program.args.length ? program.args : await file.list('streams/*.m3u') + + for (const filepath of files) { + if (!filepath.endsWith('.m3u')) continue + const playlist = await parser.parsePlaylist(filepath) + for (const stream of playlist.items) { + const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] + stream.status = status + if (status === 'error') { + const result = await checkStream(stream) + const newStatus = parseStatus(result.error) + if (status === newStatus) { + stream.remove = true + } + } + } + + const items = playlist.items + .filter(i => !i.remove) + .map(item => ({ + attrs: { + 'tvg-id': item.tvg.id, + status: item.status, + 'user-agent': item.http['user-agent'] || undefined + }, + title: item.name, + url: item.url, + vlcOpts: { + 'http-referrer': item.http.referrer || undefined, + 'http-user-agent': item.http['user-agent'] || undefined + } + })) + + const output = m3u.create(items) + await file.create(filepath, output) + } +} + +main() + +async function checkStream(item) { + const config = { + timeout: options.timeout, + delay: options.delay, + debug: options.debug + } + + const request = { + url: item.url, + http: { + referrer: item.http.referrer, + 'user-agent': item.http['user-agent'] + } + } + + return checker.check(request, config) +} + +function parseStatus(error) { + if (!error) return 'online' + + switch (error) { + case 'Operation timed out': + return 'timeout' + case 'Server returned 403 Forbidden (access denied)': + return 'blocked' + default: + return 'error' + } +} From bde30c00106ac9d8043e4b5f15942536a14de1de Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:38:30 +0300 Subject: [PATCH 09/12] Update cleaner.js --- scripts/commands/playlist/cleaner.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js index 2de731252..1eb623383 100644 --- a/scripts/commands/playlist/cleaner.js +++ b/scripts/commands/playlist/cleaner.js @@ -13,7 +13,9 @@ async function main() { for (const filepath of files) { if (!filepath.endsWith('.m3u')) continue + logger.info(`${filepath}`) const playlist = await parser.parsePlaylist(filepath) + const before = playlist.items.length for (const stream of playlist.items) { const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] stream.status = status @@ -22,6 +24,7 @@ async function main() { const newStatus = parseStatus(result.error) if (status === newStatus) { stream.remove = true + logger.info(`removed "${stream.name}"`) } } } @@ -42,8 +45,11 @@ async function main() { } })) - const output = m3u.create(items) - await file.create(filepath, output) + if (before !== items.length) { + const output = m3u.create(items) + await file.create(filepath, output) + logger.info(`saved`) + } } } From 30ad25075ada5153a1e1b2ad4faae11a431011ba Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 18:50:34 +0300 Subject: [PATCH 10/12] Update cleaner.js --- scripts/commands/playlist/cleaner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js index 1eb623383..0c2f89fab 100644 --- a/scripts/commands/playlist/cleaner.js +++ b/scripts/commands/playlist/cleaner.js @@ -19,7 +19,7 @@ async function main() { for (const stream of playlist.items) { const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] stream.status = status - if (status === 'error') { + if (status === 'error' && /^(http|https)/.test(stream.url)) { const result = await checkStream(stream) const newStatus = parseStatus(result.error) if (status === newStatus) { From 17894e71fb2f94e85e9321ed101eddd0eca3fd85 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 19:03:30 +0300 Subject: [PATCH 11/12] Update cleaner.js --- scripts/commands/playlist/cleaner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js index 0c2f89fab..3433526b3 100644 --- a/scripts/commands/playlist/cleaner.js +++ b/scripts/commands/playlist/cleaner.js @@ -19,7 +19,7 @@ async function main() { for (const stream of playlist.items) { const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] stream.status = status - if (status === 'error' && /^(http|https)/.test(stream.url)) { + if (status === 'error' && /^(http|https)/.test(stream.url) && !/\[.*\]$/.test(stream.name)) { const result = await checkStream(stream) const newStatus = parseStatus(result.error) if (status === newStatus) { From 02b27d247f3c68757d859cdcb946411700e82ffd Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 11 Mar 2022 19:10:49 +0300 Subject: [PATCH 12/12] Update cleaner.js --- scripts/commands/playlist/cleaner.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js index 3433526b3..8595dbf5f 100644 --- a/scripts/commands/playlist/cleaner.js +++ b/scripts/commands/playlist/cleaner.js @@ -1,13 +1,15 @@ const { file, parser, logger, checker, m3u } = require('../../core') const { program } = require('commander') -const options = program +program .argument('[filepath]', 'Path to file to validate') .option('-t, --timeout ', 'Set timeout for each request', parser.parseNumber, 60000) .option('-d, --delay ', 'Set delay for each request', parser.parseNumber, 0) .option('--debug', 'Enable debug mode') .parse(process.argv) +const options = program.opts() + async function main() { const files = program.args.length ? program.args : await file.list('streams/*.m3u') @@ -17,6 +19,7 @@ async function main() { const playlist = await parser.parsePlaylist(filepath) const before = playlist.items.length for (const stream of playlist.items) { + if (options.debug) logger.info(stream.url) const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] stream.status = status if (status === 'error' && /^(http|https)/.test(stream.url) && !/\[.*\]$/.test(stream.name)) {