From fae74f4e2c5e3e513f5a1f481c1cad027766357f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 1 May 2023 21:13:50 +0300 Subject: [PATCH 1/6] Update validate.js --- scripts/commands/playlist/validate.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/commands/playlist/validate.js b/scripts/commands/playlist/validate.js index 2b1b4badb..15ee4a73e 100644 --- a/scripts/commands/playlist/validate.js +++ b/scripts/commands/playlist/validate.js @@ -30,6 +30,7 @@ async function main() { const basename = file.basename(filepath) const [__, country] = basename.match(/([a-z]{2})(|_.*)\.m3u/i) || [null, null] + const buffer = {} const fileLog = [] const playlist = await parser.parsePlaylist(filepath) for (const item of playlist.items) { @@ -41,6 +42,16 @@ async function main() { }) } + if (item.url && buffer[item.url]) { + fileLog.push({ + type: 'warning', + line: item.line, + message: `"${item.url}" is already on the playlist` + }) + } else { + buffer[item.url] = true + } + const channel_id = id.generate(item.name, country) const found = blocklist.find( blocked => From 307d152410cdcd3ae9d5fd315e372e0c7912f38c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 1 May 2023 21:33:10 +0300 Subject: [PATCH 2/6] Update ru.m3u --- tests/__data__/expected/streams/ru.m3u | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__data__/expected/streams/ru.m3u b/tests/__data__/expected/streams/ru.m3u index cf2288e05..17ed7be44 100644 --- a/tests/__data__/expected/streams/ru.m3u +++ b/tests/__data__/expected/streams/ru.m3u @@ -6,4 +6,4 @@ http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 #EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 #EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) -http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 +https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?advertisingId=&appName=web&deviceId=5ca525b650be2571e3943c63 From 5b58d1bbdeaad8aa6869832348a9d3fe5f5880e3 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 1 May 2023 21:33:14 +0300 Subject: [PATCH 3/6] Update playlist_format.streams.db --- tests/__data__/input/database/playlist_format.streams.db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__data__/input/database/playlist_format.streams.db b/tests/__data__/input/database/playlist_format.streams.db index ba681f59f..7cfe4130e 100644 --- a/tests/__data__/input/database/playlist_format.streams.db +++ b/tests/__data__/input/database/playlist_format.streams.db @@ -1,4 +1,4 @@ -{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF3"} +{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?deviceId=5ca525b650be2571e3943c63&appName=web&advertisingId=","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF3"} {"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF0"} {"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF1"} {"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF2"} From fa4b1a58f3e393d00e05efae24a668d2cee6afbe Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 1 May 2023 21:33:21 +0300 Subject: [PATCH 4/6] Create url.js --- scripts/core/url.js | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/core/url.js diff --git a/scripts/core/url.js b/scripts/core/url.js new file mode 100644 index 000000000..939e42493 --- /dev/null +++ b/scripts/core/url.js @@ -0,0 +1,11 @@ +const normalize = require('normalize-url') + +const url = {} + +url.normalize = function (string) { + const normalized = normalize(string, { stripWWW: false }) + + return decodeURIComponent(normalized).replace(/\s/g, '+') +} + +module.exports = url From ee526d14f98a8de03a49af09b0355faf0d0a7ce8 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 1 May 2023 21:33:30 +0300 Subject: [PATCH 5/6] Update format.js --- scripts/commands/playlist/format.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/commands/playlist/format.js b/scripts/commands/playlist/format.js index 08f3ec921..f1c62203e 100644 --- a/scripts/commands/playlist/format.js +++ b/scripts/commands/playlist/format.js @@ -1,4 +1,5 @@ const { create: createPlaylist } = require('../../core/playlist') +const { normalize: normalizeUrl } = require('../../core/url') const { db, logger, file } = require('../../core') const { orderBy } = require('natural-orderby') const _ = require('lodash') @@ -8,6 +9,12 @@ async function main() { await db.streams.load() let streams = await db.streams.find({}) + streams = streams.map(stream => { + stream.url = normalizeUrl(stream.url) + + return stream + }) + logger.info('sorting links...') streams = orderBy( streams, From a3ce90ee675a9dba4629bd695258e6a55b5cbdb2 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 1 May 2023 21:34:16 +0300 Subject: [PATCH 6/6] Update store/setters --- scripts/commands/database/create.js | 12 ++++++------ scripts/store/setters/channel.js | 2 +- scripts/store/setters/http_referrer.js | 2 +- scripts/store/setters/index.js | 1 - scripts/store/setters/url.js | 7 ------- scripts/store/setters/user_agent.js | 2 +- 6 files changed, 9 insertions(+), 17 deletions(-) delete mode 100644 scripts/store/setters/url.js diff --git a/scripts/commands/database/create.js b/scripts/commands/database/create.js index 5cce0dd19..67c3fd440 100644 --- a/scripts/commands/database/create.js +++ b/scripts/commands/database/create.js @@ -18,12 +18,12 @@ async function main() { const stream = store.create() - stream.set('channel', { channel: item.tvg.id }) - stream.set('title', { title: item.name }) - stream.set('filepath', { filepath: item.filepath }) - stream.set('url', { url: item.url }) - stream.set('http_referrer', { http_referrer: item.http.referrer }) - stream.set('user_agent', { user_agent: item.http['user-agent'] }) + stream.set('channel', item.tvg.id) + stream.set('title', item.name) + stream.set('filepath', item.filepath) + stream.set('url', item.url) + stream.set('http_referrer', item.http.referrer) + stream.set('user_agent', item.http['user-agent']) streams.push(stream) } diff --git a/scripts/store/setters/channel.js b/scripts/store/setters/channel.js index 5d9c3ea3c..a1aed8f2e 100644 --- a/scripts/store/setters/channel.js +++ b/scripts/store/setters/channel.js @@ -1,3 +1,3 @@ -module.exports = function ({ channel }) { +module.exports = function (channel) { return channel || null } diff --git a/scripts/store/setters/http_referrer.js b/scripts/store/setters/http_referrer.js index fdbb31c46..d7cc83b24 100644 --- a/scripts/store/setters/http_referrer.js +++ b/scripts/store/setters/http_referrer.js @@ -1,3 +1,3 @@ -module.exports = function ({ http_referrer }) { +module.exports = function (http_referrer) { return http_referrer || null } diff --git a/scripts/store/setters/index.js b/scripts/store/setters/index.js index ff02bcce3..a09baefae 100644 --- a/scripts/store/setters/index.js +++ b/scripts/store/setters/index.js @@ -1,4 +1,3 @@ -exports.url = require('./url') exports.http_referrer = require('./http_referrer') exports.user_agent = require('./user_agent') exports.channel = require('./channel') diff --git a/scripts/store/setters/url.js b/scripts/store/setters/url.js deleted file mode 100644 index 995159399..000000000 --- a/scripts/store/setters/url.js +++ /dev/null @@ -1,7 +0,0 @@ -const normalize = require('normalize-url') - -module.exports = function ({ url }) { - const normalized = normalize(url, { stripWWW: false }) - - return decodeURIComponent(normalized).replace(/\s/g, '+') -} diff --git a/scripts/store/setters/user_agent.js b/scripts/store/setters/user_agent.js index ccc0c0599..38cbbaa21 100644 --- a/scripts/store/setters/user_agent.js +++ b/scripts/store/setters/user_agent.js @@ -1,3 +1,3 @@ -module.exports = function ({ user_agent }) { +module.exports = function (user_agent) { return user_agent || null }