diff --git a/scripts/commands/api/generate.ts b/scripts/commands/api/generate.ts index 9f311fd38..670fa091d 100644 --- a/scripts/commands/api/generate.ts +++ b/scripts/commands/api/generate.ts @@ -1,21 +1,37 @@ -import { Logger, Storage } from '@freearhey/core' -import { API_DIR, STREAMS_DIR } from '../../constants' +import { Logger, Storage, Collection } from '@freearhey/core' +import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants' import { PlaylistParser } from '../../core' -import { Stream } from '../../models' +import { Stream, Channel, Feed } from '../../models' +import { uniqueId } from 'lodash' async function main() { const logger = new Logger() + logger.info('loading api data...') + const dataStorage = new Storage(DATA_DIR) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => + new Feed(data).withChannel(channelsGroupedById) + ) + const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => + feed.channel ? feed.channel.id : uniqueId() + ) + logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ storage: streamsStorage }) + const parser = new PlaylistParser({ + storage: streamsStorage, + channelsGroupedById, + feedsGroupedByChannelId + }) const files = await streamsStorage.list('**/*.m3u') let streams = await parser.parse(files) streams = streams - .map(data => new Stream(data)) - .orderBy([(stream: Stream) => stream.channel]) + .orderBy((stream: Stream) => stream.getId()) .map((stream: Stream) => stream.toJSON()) - logger.info(`found ${streams.count()} streams`) logger.info('saving to .api/streams.json...') diff --git a/scripts/commands/api/load.ts b/scripts/commands/api/load.ts index 68e6e18a4..fbb1fea43 100644 --- a/scripts/commands/api/load.ts +++ b/scripts/commands/api/load.ts @@ -12,7 +12,9 @@ async function main() { client.download('countries.json'), client.download('languages.json'), client.download('regions.json'), - client.download('subdivisions.json') + client.download('subdivisions.json'), + client.download('feeds.json'), + client.download('timezones.json') ] await Promise.all(requests) diff --git a/scripts/commands/playlist/format.ts b/scripts/commands/playlist/format.ts index d532db607..8dc5dedac 100644 --- a/scripts/commands/playlist/format.ts +++ b/scripts/commands/playlist/format.ts @@ -1,25 +1,36 @@ import { Logger, Storage, Collection } from '@freearhey/core' import { STREAMS_DIR, DATA_DIR } from '../../constants' import { PlaylistParser } from '../../core' -import { Stream, Playlist, Channel } from '../../models' +import { Stream, Playlist, Channel, Feed } from '../../models' import { program } from 'commander' +import { uniqueId } from 'lodash' program.argument('[filepath]', 'Path to file to validate').parse(process.argv) async function main() { - const storage = new Storage(STREAMS_DIR) + const streamsStorage = new Storage(STREAMS_DIR) const logger = new Logger() - logger.info('loading channels from api...') + logger.info('loading data from api...') const dataStorage = new Storage(DATA_DIR) - const channelsContent = await dataStorage.json('channels.json') - const groupedChannels = new Collection(channelsContent) - .map(data => new Channel(data)) - .keyBy((channel: Channel) => channel.id) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => + new Feed(data).withChannel(channelsGroupedById) + ) + const feedsGroupedByChannelId = feeds.groupBy(feed => + feed.channel ? feed.channel.id : uniqueId() + ) logger.info('loading streams...') - const parser = new PlaylistParser({ storage }) - const files = program.args.length ? program.args : await storage.list('**/*.m3u') + const parser = new PlaylistParser({ + storage: streamsStorage, + channelsGroupedById, + feedsGroupedByChannelId + }) + const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u') let streams = await parser.parse(files) logger.info(`found ${streams.count()} streams`) @@ -35,8 +46,8 @@ async function main() { logger.info('removing wrong id...') streams = streams.map((stream: Stream) => { - if (groupedChannels.missing(stream.channel)) { - stream.channel = '' + if (!stream.channel || channelsGroupedById.missing(stream.channel.id)) { + stream.id = '' } return stream @@ -46,22 +57,22 @@ async function main() { streams = streams.orderBy( [ (stream: Stream) => stream.name, - (stream: Stream) => parseInt(stream.quality.replace('p', '')), - (stream: Stream) => stream.label, + (stream: Stream) => stream.getHorizontalResolution(), + (stream: Stream) => stream.getLabel(), (stream: Stream) => stream.url ], ['asc', 'desc', 'asc', 'asc'] ) logger.info('saving...') - const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath) + const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) for (let filepath of groupedStreams.keys()) { const streams = groupedStreams.get(filepath) || [] if (!streams.length) return const playlist = new Playlist(streams, { public: false }) - await storage.save(filepath, playlist.toString()) + await streamsStorage.save(filepath, playlist.toString()) } } diff --git a/scripts/commands/playlist/generate.ts b/scripts/commands/playlist/generate.ts index bab7839fd..46f2a2665 100644 --- a/scripts/commands/playlist/generate.ts +++ b/scripts/commands/playlist/generate.ts @@ -1,14 +1,23 @@ -import { Logger, Storage, Collection, File } from '@freearhey/core' +import { Logger, Storage, Collection } from '@freearhey/core' import { PlaylistParser } from '../../core' -import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models' -import _ from 'lodash' +import { + Stream, + Category, + Channel, + Language, + Country, + Region, + Subdivision, + Feed, + Timezone +} from '../../models' +import { uniqueId } from 'lodash' import { CategoriesGenerator, CountriesGenerator, LanguagesGenerator, RegionsGenerator, IndexGenerator, - IndexNsfwGenerator, IndexCategoryGenerator, IndexCountryGenerator, IndexLanguageGenerator, @@ -19,123 +28,134 @@ import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants' async function main() { const logger = new Logger() const dataStorage = new Storage(DATA_DIR) + const generatorsLogger = new Logger({ + stream: await new Storage(LOGS_DIR).createStream(`generators.log`) + }) logger.info('loading data from api...') - const channelsContent = await dataStorage.json('channels.json') - const channels = new Collection(channelsContent).map(data => new Channel(data)) - const categoriesContent = await dataStorage.json('categories.json') - const categories = new Collection(categoriesContent).map(data => new Category(data)) - const countriesContent = await dataStorage.json('countries.json') - const countries = new Collection(countriesContent).map(data => new Country(data)) - const languagesContent = await dataStorage.json('languages.json') - const languages = new Collection(languagesContent).map(data => new Language(data)) - const regionsContent = await dataStorage.json('regions.json') - const regions = new Collection(regionsContent).map(data => new Region(data)) - const subdivisionsContent = await dataStorage.json('subdivisions.json') - const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data)) + const categoriesData = await dataStorage.json('categories.json') + const countriesData = await dataStorage.json('countries.json') + const languagesData = await dataStorage.json('languages.json') + const regionsData = await dataStorage.json('regions.json') + const subdivisionsData = await dataStorage.json('subdivisions.json') + const timezonesData = await dataStorage.json('timezones.json') + const channelsData = await dataStorage.json('channels.json') + const feedsData = await dataStorage.json('feeds.json') + + logger.info('preparing data...') + const subdivisions = new Collection(subdivisionsData).map(data => new Subdivision(data)) + const subdivisionsGroupedByCode = subdivisions.keyBy( + (subdivision: Subdivision) => subdivision.code + ) + const subdivisionsGroupedByCountryCode = subdivisions.groupBy( + (subdivision: Subdivision) => subdivision.countryCode + ) + let regions = new Collection(regionsData).map(data => + new Region(data).withSubdivisions(subdivisions) + ) + const regionsGroupedByCode = regions.keyBy((region: Region) => region.code) + const categories = new Collection(categoriesData).map(data => new Category(data)) + const categoriesGroupedById = categories.keyBy((category: Category) => category.id) + const languages = new Collection(languagesData).map(data => new Language(data)) + const languagesGroupedByCode = languages.keyBy((language: Language) => language.code) + const countries = new Collection(countriesData).map(data => + new Country(data) + .withRegions(regions) + .withLanguage(languagesGroupedByCode) + .withSubdivisions(subdivisionsGroupedByCountryCode) + ) + const countriesGroupedByCode = countries.keyBy((country: Country) => country.code) + regions = regions.map((region: Region) => region.withCountries(countriesGroupedByCode)) + + const timezones = new Collection(timezonesData).map(data => + new Timezone(data).withCountries(countriesGroupedByCode) + ) + const timezonesGroupedById = timezones.keyBy((timezone: Timezone) => timezone.id) + + const channels = new Collection(channelsData).map(data => + new Channel(data) + .withCategories(categoriesGroupedById) + .withCountry(countriesGroupedByCode) + .withSubdivision(subdivisionsGroupedByCode) + ) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feeds = new Collection(feedsData).map(data => + new Feed(data) + .withChannel(channelsGroupedById) + .withLanguages(languagesGroupedByCode) + .withTimezones(timezonesGroupedById) + .withBroadcastCountries( + countriesGroupedByCode, + regionsGroupedByCode, + subdivisionsGroupedByCode + ) + .withBroadcastRegions(regions, regionsGroupedByCode) + .withBroadcastSubdivisions(subdivisionsGroupedByCode) + ) + const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => + feed.channel ? feed.channel.id : uniqueId() + ) logger.info('loading streams...') - let streams = await loadStreams({ channels, categories, languages }) - let totalStreams = streams.count() - streams = streams.uniqBy((stream: Stream) => (stream.channel || _.uniqueId()) + stream.timeshift) + const storage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ + storage, + channelsGroupedById, + feedsGroupedByChannelId + }) + const files = await storage.list('**/*.m3u') + let streams = await parser.parse(files) + const totalStreams = streams.count() + streams = streams.uniqBy((stream: Stream) => stream.getId() || uniqueId()) logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`) - const generatorsLogger = new Logger({ - stream: await new Storage(LOGS_DIR).createStream(`generators.log`) - }) + logger.info('sorting streams...') + streams = streams.orderBy( + [ + (stream: Stream) => stream.getId(), + (stream: Stream) => stream.getHorizontalResolution(), + (stream: Stream) => stream.getLabel() + ], + ['asc', 'asc', 'desc'] + ) logger.info('generating categories/...') await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate() + logger.info('generating countries/...') await new CountriesGenerator({ countries, streams, - regions, - subdivisions, logger: generatorsLogger }).generate() + logger.info('generating languages/...') await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate() + logger.info('generating regions/...') await new RegionsGenerator({ streams, regions, - subdivisions, logger: generatorsLogger }).generate() + logger.info('generating index.m3u...') await new IndexGenerator({ streams, logger: generatorsLogger }).generate() + logger.info('generating index.category.m3u...') await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate() + logger.info('generating index.country.m3u...') await new IndexCountryGenerator({ streams, - countries, - regions, - subdivisions, logger: generatorsLogger }).generate() + logger.info('generating index.language.m3u...') await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate() + logger.info('generating index.region.m3u...') await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate() } main() - -async function loadStreams({ - channels, - categories, - languages -}: { - channels: Collection - categories: Collection - languages: Collection -}) { - const groupedChannels = channels.keyBy(channel => channel.id) - const groupedCategories = categories.keyBy(category => category.id) - const groupedLanguages = languages.keyBy(language => language.code) - - const storage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ storage }) - const files = await storage.list('**/*.m3u') - let streams = await parser.parse(files) - - streams = streams - .orderBy( - [ - (stream: Stream) => stream.channel, - (stream: Stream) => parseInt(stream.quality.replace('p', '')), - (stream: Stream) => stream.label - ], - ['asc', 'asc', 'desc', 'asc'] - ) - .map((stream: Stream) => { - const channel: Channel | undefined = groupedChannels.get(stream.channel) - - if (channel) { - const channelCategories = channel.categories - .map((id: string) => groupedCategories.get(id)) - .filter(Boolean) - const channelLanguages = channel.languages - .map((id: string) => groupedLanguages.get(id)) - .filter(Boolean) - - stream.categories = channelCategories - stream.languages = channelLanguages - stream.broadcastArea = channel.broadcastArea - stream.isNSFW = channel.isNSFW - if (channel.logo) stream.logo = channel.logo - } else { - const file = new File(stream.filepath) - const [_, countryCode] = file.name().match(/^([a-z]{2})(_|$)/) || [null, null] - const defaultBroadcastArea = countryCode ? [`c/${countryCode.toUpperCase()}`] : [] - - stream.broadcastArea = new Collection(defaultBroadcastArea) - } - - return stream - }) - - return streams -} diff --git a/scripts/commands/playlist/test.ts b/scripts/commands/playlist/test.ts index 74c19e66c..f32f2e0c2 100644 --- a/scripts/commands/playlist/test.ts +++ b/scripts/commands/playlist/test.ts @@ -1,7 +1,7 @@ import { Logger, Storage, Collection } from '@freearhey/core' -import { ROOT_DIR, STREAMS_DIR } from '../../constants' +import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants' import { PlaylistParser, StreamTester, CliTable } from '../../core' -import { Stream } from '../../models' +import { Stream, Feed, Channel } from '../../models' import { program } from 'commander' import { eachLimit } from 'async-es' import commandExists from 'command-exists' @@ -38,8 +38,6 @@ const logger = new Logger() const tester = new StreamTester() async function main() { - const storage = new Storage(ROOT_DIR) - if (await isOffline()) { logger.error(chalk.red('Internet connection is required for the script to work')) @@ -56,9 +54,25 @@ async function main() { return } + logger.info('loading channels from api...') + const dataStorage = new Storage(DATA_DIR) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => + new Feed(data).withChannel(channelsGroupedById) + ) + const feedsGroupedByChannelId = feeds.groupBy(feed => feed.channel) + logger.info('loading streams...') - const parser = new PlaylistParser({ storage }) - const files = program.args.length ? program.args : await storage.list(`${STREAMS_DIR}/*.m3u`) + const rootStorage = new Storage(ROOT_DIR) + const parser = new PlaylistParser({ + storage: rootStorage, + channelsGroupedById, + feedsGroupedByChannelId + }) + const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`) streams = await parser.parse(files) logger.info(`found ${streams.count()} streams`) @@ -89,7 +103,7 @@ async function main() { main() async function runTest(stream: Stream) { - const key = stream.filepath + stream.channel + stream.url + const key = stream.filepath + stream.getId() + stream.url results[key] = chalk.white('LOADING...') const result = await tester.test(stream) @@ -125,11 +139,11 @@ function drawTable() { ] }) streams.forEach((stream: Stream, index: number) => { - const status = results[stream.filepath + stream.channel + stream.url] || chalk.gray('PENDING') + const status = results[stream.filepath + stream.getId() + stream.url] || chalk.gray('PENDING') const row = { '': index, - 'tvg-id': stream.channel.length > 25 ? stream.channel.slice(0, 22) + '...' : stream.channel, + 'tvg-id': stream.getId().length > 25 ? stream.getId().slice(0, 22) + '...' : stream.getId(), url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url, status } diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index 1fb56ee03..b2ac5b814 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -1,45 +1,63 @@ import { Logger, Storage, Collection, Dictionary } from '@freearhey/core' import { DATA_DIR, STREAMS_DIR } from '../../constants' import { IssueLoader, PlaylistParser } from '../../core' -import { Stream, Playlist, Channel, Issue } from '../../models' +import { Stream, Playlist, Channel, Feed, Issue } from '../../models' import validUrl from 'valid-url' +import { uniqueId } from 'lodash' let processedIssues = new Collection() -let streams: Collection -let groupedChannels: Dictionary -let issues: Collection async function main() { const logger = new Logger({ disabled: true }) const loader = new IssueLoader() logger.info('loading issues...') - issues = await loader.load() + const issues = await loader.load() logger.info('loading channels from api...') const dataStorage = new Storage(DATA_DIR) - const channelsContent = await dataStorage.json('channels.json') - groupedChannels = new Collection(channelsContent) - .map(data => new Channel(data)) - .keyBy((channel: Channel) => channel.id) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => + new Feed(data).withChannel(channelsGroupedById) + ) + const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => + feed.channel ? feed.channel.id : uniqueId() + ) logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ storage: streamsStorage }) + const parser = new PlaylistParser({ + storage: streamsStorage, + feedsGroupedByChannelId, + channelsGroupedById + }) const files = await streamsStorage.list('**/*.m3u') - streams = await parser.parse(files) + const streams = await parser.parse(files) logger.info('removing broken streams...') - await removeStreams(loader) + await removeStreams({ streams, issues }) logger.info('edit stream description...') - await editStreams(loader) + await editStreams({ + streams, + issues, + channelsGroupedById, + feedsGroupedByChannelId + }) logger.info('add new streams...') - await addStreams(loader) + await addStreams({ + streams, + issues, + channelsGroupedById, + feedsGroupedByChannelId + }) logger.info('saving...') - const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath) + const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath()) for (let filepath of groupedStreams.keys()) { let streams = groupedStreams.get(filepath) || [] streams = streams.filter((stream: Stream) => stream.removed === false) @@ -54,7 +72,7 @@ async function main() { main() -async function removeStreams(loader: IssueLoader) { +async function removeStreams({ streams, issues }: { streams: Collection; issues: Collection }) { const requests = issues.filter( issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved') ) @@ -62,22 +80,35 @@ async function removeStreams(loader: IssueLoader) { const data = issue.data if (data.missing('brokenLinks')) return - const brokenLinks = data.getString('brokenLinks').split(/\r?\n/).filter(Boolean) + const brokenLinks = data.getString('brokenLinks') || '' let changed = false - brokenLinks.forEach(link => { - const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) - if (found) { - found.removed = true - changed = true - } - }) + brokenLinks + .split(/\r?\n/) + .filter(Boolean) + .forEach(link => { + const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim()) + if (found) { + found.removed = true + changed = true + } + }) if (changed) processedIssues.add(issue.number) }) } -async function editStreams(loader: IssueLoader) { +async function editStreams({ + streams, + issues, + channelsGroupedById, + feedsGroupedByChannelId +}: { + streams: Collection + issues: Collection + channelsGroupedById: Dictionary + feedsGroupedByChannelId: Dictionary +}) { const requests = issues.filter( issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved') ) @@ -86,59 +117,110 @@ async function editStreams(loader: IssueLoader) { if (data.missing('streamUrl')) return - let stream = streams.first( + let stream: Stream = streams.first( (_stream: Stream) => _stream.url === data.getString('streamUrl') - ) as Stream - + ) if (!stream) return - if (data.has('channelId')) { - const channel = groupedChannels.get(data.getString('channelId')) - - if (!channel) return - - stream.channel = data.getString('channelId') - stream.filepath = `${channel.country.toLowerCase()}.m3u` - stream.line = -1 - stream.name = channel.name + const streamId = data.getString('streamId') || '' + const [channelId, feedId] = streamId.split('@') + + if (channelId) { + stream + .setChannelId(channelId) + .setFeedId(feedId) + .withChannel(channelsGroupedById) + .withFeed(feedsGroupedByChannelId) + .updateId() + .updateName() + .updateFilepath() } - if (data.has('label')) stream.label = data.getString('label') - if (data.has('quality')) stream.quality = data.getString('quality') - if (data.has('httpUserAgent')) stream.httpUserAgent = data.getString('httpUserAgent') - if (data.has('httpReferrer')) stream.httpReferrer = data.getString('httpReferrer') + const label = data.getString('label') || '' + const quality = data.getString('quality') || '' + const httpUserAgent = data.getString('httpUserAgent') || '' + const httpReferrer = data.getString('httpReferrer') || '' + + if (data.has('label')) stream.setLabel(label) + if (data.has('quality')) stream.setQuality(quality) + if (data.has('httpUserAgent')) stream.setHttpUserAgent(httpUserAgent) + if (data.has('httpReferrer')) stream.setHttpReferrer(httpReferrer) processedIssues.add(issue.number) }) } -async function addStreams(loader: IssueLoader) { +async function addStreams({ + streams, + issues, + channelsGroupedById, + feedsGroupedByChannelId +}: { + streams: Collection + issues: Collection + channelsGroupedById: Dictionary + feedsGroupedByChannelId: Dictionary +}) { const requests = issues.filter( issue => issue.labels.includes('streams:add') && issue.labels.includes('approved') ) requests.forEach((issue: Issue) => { const data = issue.data - if (data.missing('channelId') || data.missing('streamUrl')) return + if (data.missing('streamId') || data.missing('streamUrl')) return if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return - if (!validUrl.isUri(data.getString('streamUrl'))) return + const stringUrl = data.getString('streamUrl') || '' + if (!isUri(stringUrl)) return - const channel = groupedChannels.get(data.getString('channelId')) + const streamId = data.getString('streamId') || '' + const [channelId] = streamId.split('@') + const channel: Channel = channelsGroupedById.get(channelId) if (!channel) return + const label = data.getString('label') || '' + const quality = data.getString('quality') || '' + const httpUserAgent = data.getString('httpUserAgent') || '' + const httpReferrer = data.getString('httpReferrer') || '' + const stream = new Stream({ - channel: data.getString('channelId'), - url: data.getString('streamUrl'), - label: data.getString('label'), - quality: data.getString('quality'), - httpUserAgent: data.getString('httpUserAgent'), - httpReferrer: data.getString('httpReferrer'), - filepath: `${channel.country.toLowerCase()}.m3u`, + tvg: { + id: streamId, + name: '', + url: '', + logo: '', + rec: '', + shift: '' + }, + name: data.getString('channelName') || channel.name, + url: stringUrl, + group: { + title: '' + }, + http: { + 'user-agent': httpUserAgent, + referrer: httpReferrer + }, line: -1, - name: data.getString('channelName') || channel.name + raw: '', + timeshift: '', + catchup: { + type: '', + source: '', + days: '' + } }) + .withChannel(channelsGroupedById) + .withFeed(feedsGroupedByChannelId) + .setLabel(label) + .setQuality(quality) + .updateName() + .updateFilepath() streams.add(stream) processedIssues.add(issue.number) }) } + +function isUri(string: string) { + return validUrl.isUri(encodeURI(string)) +} diff --git a/scripts/commands/playlist/validate.ts b/scripts/commands/playlist/validate.ts index b0c140dcf..6296b5651 100644 --- a/scripts/commands/playlist/validate.ts +++ b/scripts/commands/playlist/validate.ts @@ -1,9 +1,9 @@ import { Logger, Storage, Collection, Dictionary } from '@freearhey/core' import { PlaylistParser } from '../../core' -import { Channel, Stream, Blocked } from '../../models' +import { Channel, Stream, Blocked, Feed } from '../../models' import { program } from 'commander' import chalk from 'chalk' -import _ from 'lodash' +import { uniqueId } from 'lodash' import { DATA_DIR, STREAMS_DIR } from '../../constants' program.argument('[filepath]', 'Path to file to validate').parse(process.argv) @@ -17,41 +17,52 @@ type LogItem = { async function main() { const logger = new Logger() - logger.info(`loading blocklist...`) + logger.info('loading data from api...') const dataStorage = new Storage(DATA_DIR) - const channelsContent = await dataStorage.json('channels.json') - const channels = new Collection(channelsContent).map(data => new Channel(data)) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => + new Feed(data).withChannel(channelsGroupedById) + ) + const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => + feed.channel ? feed.channel.id : uniqueId() + ) const blocklistContent = await dataStorage.json('blocklist.json') const blocklist = new Collection(blocklistContent).map(data => new Blocked(data)) - - logger.info(`found ${blocklist.count()} records`) + const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId) logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ storage: streamsStorage }) + const parser = new PlaylistParser({ + storage: streamsStorage, + channelsGroupedById, + feedsGroupedByChannelId + }) const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u') const streams = await parser.parse(files) - logger.info(`found ${streams.count()} streams`) let errors = new Collection() let warnings = new Collection() - let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath) - for (const filepath of groupedStreams.keys()) { - const streams = groupedStreams.get(filepath) + let streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath()) + for (const filepath of streamsGroupedByFilepath.keys()) { + const streams = streamsGroupedByFilepath.get(filepath) if (!streams) continue const log = new Collection() const buffer = new Dictionary() streams.forEach((stream: Stream) => { - const invalidId = - stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel) - if (invalidId) { - log.add({ - type: 'warning', - line: stream.line, - message: `"${stream.channel}" is not in the database` - }) + if (stream.channelId) { + const channel = channelsGroupedById.get(stream.channelId) + if (!channel) { + log.add({ + type: 'warning', + line: stream.line, + message: `"${stream.id}" is not in the database` + }) + } } const duplicate = stream.url && buffer.has(stream.url) @@ -65,19 +76,19 @@ async function main() { buffer.set(stream.url, true) } - const blocked = blocklist.first(blocked => stream.channel === blocked.channel) + const blocked = stream.channel ? blocklistGroupedByChannelId.get(stream.channel.id) : false if (blocked) { if (blocked.reason === 'dmca') { log.add({ type: 'error', line: stream.line, - message: `"${stream.channel}" is on the blocklist due to claims of copyright holders (${blocked.ref})` + message: `"${blocked.channelId}" is on the blocklist due to claims of copyright holders (${blocked.ref})` }) } else if (blocked.reason === 'nsfw') { log.add({ type: 'error', line: stream.line, - message: `"${stream.channel}" is on the blocklist due to NSFW content (${blocked.ref})` + message: `"${blocked.channelId}" is on the blocklist due to NSFW content (${blocked.ref})` }) } } diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index 6b440547b..7584105d1 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -1,154 +1,164 @@ import { Logger, Storage, Collection, Dictionary } from '@freearhey/core' import { DATA_DIR, STREAMS_DIR } from '../../constants' import { IssueLoader, PlaylistParser } from '../../core' -import { Blocked, Channel, Issue, Stream } from '../../models' +import { Blocked, Channel, Issue, Stream, Feed } from '../../models' +import { uniqueId } from 'lodash' async function main() { const logger = new Logger() const loader = new IssueLoader() - - const storage = new Storage(DATA_DIR) + let report = new Collection() logger.info('loading issues...') const issues = await loader.load() + logger.info('loading data from api...') + const dataStorage = new Storage(DATA_DIR) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => + new Feed(data).withChannel(channelsGroupedById) + ) + const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => + feed.channel ? feed.channel.id : uniqueId() + ) + const blocklistContent = await dataStorage.json('blocklist.json') + const blocklist = new Collection(blocklistContent).map(data => new Blocked(data)) + const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId) + logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ storage: streamsStorage }) + const parser = new PlaylistParser({ + storage: streamsStorage, + channelsGroupedById, + feedsGroupedByChannelId + }) const files = await streamsStorage.list('**/*.m3u') const streams = await parser.parse(files) const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url) - const streamsGroupedByChannel = streams.groupBy((stream: Stream) => stream.channel) + const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId) + + logger.info('checking broken streams reports...') + const brokenStreamReports = issues.filter(issue => + issue.labels.find((label: string) => label === 'broken stream') + ) + brokenStreamReports.forEach((issue: Issue) => { + const brokenLinks = issue.data.getArray('brokenLinks') || [] + + if (!brokenLinks.length) { + const result = { + issueNumber: issue.number, + type: 'broken stream', + streamId: undefined, + streamUrl: undefined, + status: 'missing_link' + } - logger.info('loading channels from api...') - const channelsContent = await storage.json('channels.json') - const channelsGroupedById = new Collection(channelsContent) - .map(data => new Channel(data)) - .groupBy((channel: Channel) => channel.id) + report.add(result) + } else { + for (const streamUrl of brokenLinks) { + const result = { + issueNumber: issue.number, + type: 'broken stream', + streamId: undefined, + streamUrl: truncate(streamUrl), + status: 'pending' + } - logger.info('loading blocklist from api...') - const blocklistContent = await storage.json('blocklist.json') - const blocklistGroupedByChannel = new Collection(blocklistContent) - .map(data => new Blocked(data)) - .groupBy((blocked: Blocked) => blocked.channel) + if (streamsGroupedByUrl.missing(streamUrl)) { + result.status = 'wrong_link' + } - let report = new Collection() + report.add(result) + } + } + }) logger.info('checking streams:add requests...') const addRequests = issues.filter(issue => issue.labels.includes('streams:add')) const addRequestsBuffer = new Dictionary() addRequests.forEach((issue: Issue) => { - const channelId = issue.data.getString('channelId') || undefined - const streamUrl = issue.data.getString('streamUrl') + const streamId = issue.data.getString('streamId') || '' + const streamUrl = issue.data.getString('streamUrl') || '' + const [channelId] = streamId.split('@') - const result = new Dictionary({ + const result = { issueNumber: issue.number, type: 'streams:add', - channelId, - streamUrl, + streamId: streamId || undefined, + streamUrl: truncate(streamUrl), status: 'pending' - }) + } - if (!channelId) result.set('status', 'missing_id') - else if (!streamUrl) result.set('status', 'missing_link') - else if (blocklistGroupedByChannel.has(channelId)) result.set('status', 'blocked') - else if (channelsGroupedById.missing(channelId)) result.set('status', 'wrong_id') - else if (streamsGroupedByUrl.has(streamUrl)) result.set('status', 'on_playlist') - else if (addRequestsBuffer.has(streamUrl)) result.set('status', 'duplicate') - else result.set('status', 'pending') + if (!channelId) result.status = 'missing_id' + else if (!streamUrl) result.status = 'missing_link' + else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked' + else if (channelsGroupedById.missing(channelId)) result.status = 'wrong_id' + else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist' + else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate' + else result.status = 'pending' addRequestsBuffer.set(streamUrl, true) - report.add(result.data()) + report.add(result) }) logger.info('checking streams:edit requests...') - const editRequests = issues.filter(issue => issue.labels.find(label => label === 'streams:edit')) + const editRequests = issues.filter(issue => + issue.labels.find((label: string) => label === 'streams:edit') + ) editRequests.forEach((issue: Issue) => { - const channelId = issue.data.getString('channelId') || undefined - const streamUrl = issue.data.getString('streamUrl') || undefined + const streamId = issue.data.getString('streamId') || '' + const streamUrl = issue.data.getString('streamUrl') || '' + const [channelId] = streamId.split('@') - const result = new Dictionary({ + const result = { issueNumber: issue.number, type: 'streams:edit', - channelId, - streamUrl, + streamId: streamId || undefined, + streamUrl: truncate(streamUrl), status: 'pending' - }) - - if (!streamUrl) result.set('status', 'missing_link') - else if (streamsGroupedByUrl.missing(streamUrl)) result.set('status', 'invalid_link') - else if (channelId && channelsGroupedById.missing(channelId)) result.set('status', 'invalid_id') - - report.add(result.data()) - }) - - logger.info('checking broken streams reports...') - const brokenStreamReports = issues.filter(issue => - issue.labels.find(label => label === 'broken stream') - ) - brokenStreamReports.forEach((issue: Issue) => { - const brokenLinks = issue.data.getArray('brokenLinks') || [] - - if (!brokenLinks.length) { - const result = new Dictionary({ - issueNumber: issue.number, - type: 'broken stream', - channelId: undefined, - streamUrl: undefined, - status: 'missing_link' - }) + } - report.add(result.data()) - } else { - for (const streamUrl of brokenLinks) { - const result = new Dictionary({ - issueNumber: issue.number, - type: 'broken stream', - channelId: undefined, - streamUrl: undefined, - status: 'pending' - }) + if (!streamUrl) result.status = 'missing_link' + else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link' + else if (channelId && channelsGroupedById.missing(channelId)) result.status = 'invalid_id' - if (streamsGroupedByUrl.missing(streamUrl)) { - result.set('streamUrl', streamUrl) - result.set('status', 'wrong_link') - } - - report.add(result.data()) - } - } + report.add(result) }) logger.info('checking channel search requests...') const channelSearchRequests = issues.filter(issue => - issue.labels.find(label => label === 'channel search') + issue.labels.find((label: string) => label === 'channel search') ) const channelSearchRequestsBuffer = new Dictionary() channelSearchRequests.forEach((issue: Issue) => { - const channelId = issue.data.getString('channelId') + const streamId = issue.data.getString('channelId') || '' + const [channelId] = streamId.split('@') - const result = new Dictionary({ + const result = { issueNumber: issue.number, type: 'channel search', - channelId, + streamId: streamId || undefined, streamUrl: undefined, status: 'pending' - }) + } - if (!channelId) result.set('status', 'missing_id') - else if (channelsGroupedById.missing(channelId)) result.set('status', 'invalid_id') - else if (channelSearchRequestsBuffer.has(channelId)) result.set('status', 'duplicate') - else if (blocklistGroupedByChannel.has(channelId)) result.set('status', 'blocked') - else if (streamsGroupedByChannel.has(channelId)) result.set('status', 'fulfilled') + if (!channelId) result.status = 'missing_id' + else if (channelsGroupedById.missing(channelId)) result.status = 'invalid_id' + else if (channelSearchRequestsBuffer.has(channelId)) result.status = 'duplicate' + else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked' + else if (streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled' else { const channelData = channelsGroupedById.get(channelId) - if (channelData.length && channelData[0].closed) result.set('status', 'closed') + if (channelData.length && channelData[0].closed) result.status = 'closed' } channelSearchRequestsBuffer.set(channelId, true) - report.add(result.data()) + report.add(result) }) report = report.orderBy(item => item.issueNumber).filter(item => item.status !== 'pending') @@ -157,3 +167,10 @@ async function main() { } main() + +function truncate(string: string, limit: number = 100) { + if (!string) return string + if (string.length < limit) return string + + return string.slice(0, limit) + '...' +} diff --git a/scripts/core/apiClient.ts b/scripts/core/apiClient.ts index 66fa28a87..3b6291908 100644 --- a/scripts/core/apiClient.ts +++ b/scripts/core/apiClient.ts @@ -41,7 +41,7 @@ export class ApiClient { } async download(filename: string) { - const stream = await this.storage.createStream(`/temp/data/${filename}`) + const stream = await this.storage.createStream(`temp/data/${filename}`) const bar = this.progressBar.create(0, 0, { filename }) diff --git a/scripts/core/cliTable.ts b/scripts/core/cliTable.ts index 4d1fe3253..61d9e608e 100644 --- a/scripts/core/cliTable.ts +++ b/scripts/core/cliTable.ts @@ -1,9 +1,10 @@ import { Table } from 'console-table-printer' +import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table' export class CliTable { table: Table - constructor(options?) { + constructor(options?: ComplexOptions | string[]) { this.table = new Table(options) } diff --git a/scripts/core/issueData.ts b/scripts/core/issueData.ts index ee8918b65..61123f4aa 100644 --- a/scripts/core/issueData.ts +++ b/scripts/core/issueData.ts @@ -18,7 +18,7 @@ export class IssueData { return Boolean(this._data.get(key)) } - getString(key: string): string { + getString(key: string): string | undefined { const deleteSymbol = '~' return this._data.get(key) === deleteSymbol ? '' : this._data.get(key) diff --git a/scripts/core/issueLoader.ts b/scripts/core/issueLoader.ts index 535e2e744..1594eeb37 100644 --- a/scripts/core/issueLoader.ts +++ b/scripts/core/issueLoader.ts @@ -16,7 +16,7 @@ export class IssueLoader { } let issues: object[] = [] if (TESTING) { - issues = (await import('../../tests/__data__/input/issues/all.js')).default + issues = (await import('../../tests/__data__/input/playlist_update/issues.js')).default } else { issues = await octokit.paginate(octokit.rest.issues.listForRepo, { owner: OWNER, diff --git a/scripts/core/issueParser.ts b/scripts/core/issueParser.ts index e43f505d0..61e61c6d9 100644 --- a/scripts/core/issueParser.ts +++ b/scripts/core/issueParser.ts @@ -3,11 +3,10 @@ import { Issue } from '../models' import { IssueData } from './issueData' const FIELDS = new Dictionary({ + 'Stream ID': 'streamId', 'Channel ID': 'channelId', - 'Channel ID (required)': 'channelId', + 'Feed ID': 'feedId', 'Stream URL': 'streamUrl', - 'Stream URL (optional)': 'streamUrl', - 'Stream URL (required)': 'streamUrl', 'Broken Link': 'brokenLinks', 'Broken Links': 'brokenLinks', Label: 'label', @@ -18,8 +17,7 @@ const FIELDS = new Dictionary({ 'HTTP Referrer': 'httpReferrer', 'What happened to the stream?': 'reason', Reason: 'reason', - Notes: 'notes', - 'Notes (optional)': 'notes' + Notes: 'notes' }) export class IssueParser { @@ -30,7 +28,7 @@ export class IssueParser { fields.forEach((field: string) => { const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : [] let _label = parsed.shift() - _label = _label ? _label.trim() : '' + _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : '' let _value = parsed.join('\r\n') _value = _value ? _value.trim() : '' diff --git a/scripts/core/logParser.ts b/scripts/core/logParser.ts index 322858e0c..fb888f6b5 100644 --- a/scripts/core/logParser.ts +++ b/scripts/core/logParser.ts @@ -1,4 +1,5 @@ export type LogItem = { + type: string filepath: string count: number } diff --git a/scripts/core/playlistParser.ts b/scripts/core/playlistParser.ts index d615f5c6c..b28876663 100644 --- a/scripts/core/playlistParser.ts +++ b/scripts/core/playlistParser.ts @@ -1,12 +1,22 @@ -import { Collection, Storage } from '@freearhey/core' +import { Collection, Storage, Dictionary } from '@freearhey/core' import parser from 'iptv-playlist-parser' import { Stream } from '../models' +type PlaylistPareserProps = { + storage: Storage + feedsGroupedByChannelId: Dictionary + channelsGroupedById: Dictionary +} + export class PlaylistParser { storage: Storage + feedsGroupedByChannelId: Dictionary + channelsGroupedById: Dictionary - constructor({ storage }: { storage: Storage }) { + constructor({ storage, feedsGroupedByChannelId, channelsGroupedById }: PlaylistPareserProps) { this.storage = storage + this.feedsGroupedByChannelId = feedsGroupedByChannelId + this.channelsGroupedById = channelsGroupedById } async parse(files: string[]): Promise { @@ -21,41 +31,18 @@ export class PlaylistParser { } async parseFile(filepath: string): Promise { - const streams = new Collection() - const content = await this.storage.load(filepath) const parsed: parser.Playlist = parser.parse(content) - parsed.items.forEach((item: parser.PlaylistItem) => { - const { name, label, quality } = parseTitle(item.name) - const stream = new Stream({ - channel: item.tvg.id, - name, - label, - quality, - filepath, - line: item.line, - url: item.url, - httpReferrer: item.http.referrer, - httpUserAgent: item.http['user-agent'] - }) - - streams.add(stream) + const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => { + const stream = new Stream(data) + .withFeed(this.feedsGroupedByChannelId) + .withChannel(this.channelsGroupedById) + .setFilepath(filepath) + + return stream }) return streams } } - -function parseTitle(title: string): { name: string; label: string; quality: string } { - const [, label] = title.match(/ \[(.*)\]$/) || [null, ''] - title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '') - const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, ''] - title = title.replace(new RegExp(` \\(${quality}\\)$`), '') - - return { name: title, label, quality } -} - -function escapeRegExp(text) { - return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') -} diff --git a/scripts/core/streamTester.ts b/scripts/core/streamTester.ts index d3f772f97..89c44de74 100644 --- a/scripts/core/streamTester.ts +++ b/scripts/core/streamTester.ts @@ -11,15 +11,15 @@ export class StreamTester { async test(stream: Stream) { if (TESTING) { - const results = (await import('../../tests/__data__/input/test_results/all.js')).default + const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default return results[stream.url] } else { return this.checker.checkStream({ url: stream.url, http: { - referrer: stream.httpReferrer, - 'user-agent': stream.httpUserAgent + referrer: stream.getHttpReferrer(), + 'user-agent': stream.getHttpUserAgent() } }) } diff --git a/scripts/generators/categoriesGenerator.ts b/scripts/generators/categoriesGenerator.ts index 672af3d90..cd20b6ea4 100644 --- a/scripts/generators/categoriesGenerator.ts +++ b/scripts/generators/categoriesGenerator.ts @@ -29,11 +29,7 @@ export class CategoriesGenerator implements Generator { const categoryStreams = streams .filter((stream: Stream) => stream.hasCategory(category)) .map((stream: Stream) => { - const streamCategories = stream.categories - .map((category: Category) => category.name) - .sort() - const groupTitle = stream.categories ? streamCategories.join(';') : '' - stream.groupTitle = groupTitle + stream.groupTitle = stream.getCategoryNames().join(';') return stream }) @@ -41,13 +37,17 @@ export class CategoriesGenerator implements Generator { const playlist = new Playlist(categoryStreams, { public: true }) const filepath = `categories/${category.id}.m3u` await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + ) }) - const undefinedStreams = streams.filter((stream: Stream) => stream.noCategories()) + const undefinedStreams = streams.filter((stream: Stream) => !stream.hasCategories()) const playlist = new Playlist(undefinedStreams, { public: true }) const filepath = 'categories/undefined.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + ) } } diff --git a/scripts/generators/countriesGenerator.ts b/scripts/generators/countriesGenerator.ts index 0b5bc8c11..c935da5a4 100644 --- a/scripts/generators/countriesGenerator.ts +++ b/scripts/generators/countriesGenerator.ts @@ -1,12 +1,10 @@ import { Generator } from './generator' import { Collection, Storage, Logger } from '@freearhey/core' -import { Country, Region, Subdivision, Stream, Playlist } from '../models' +import { Country, Subdivision, Stream, Playlist } from '../models' import { PUBLIC_DIR } from '../constants' type CountriesGeneratorProps = { streams: Collection - regions: Collection - subdivisions: Collection countries: Collection logger: Logger } @@ -14,55 +12,37 @@ type CountriesGeneratorProps = { export class CountriesGenerator implements Generator { streams: Collection countries: Collection - regions: Collection - subdivisions: Collection storage: Storage logger: Logger - constructor({ streams, countries, regions, subdivisions, logger }: CountriesGeneratorProps) { + constructor({ streams, countries, logger }: CountriesGeneratorProps) { this.streams = streams this.countries = countries - this.regions = regions - this.subdivisions = subdivisions this.storage = new Storage(PUBLIC_DIR) this.logger = logger } async generate(): Promise { const streams = this.streams - .orderBy([stream => stream.getTitle()]) + .orderBy((stream: Stream) => stream.getTitle()) .filter((stream: Stream) => stream.isSFW()) - const regions = this.regions.filter((region: Region) => region.code !== 'INT') this.countries.forEach(async (country: Country) => { - const countrySubdivisions = this.subdivisions.filter( - (subdivision: Subdivision) => subdivision.country === country.code + const countryStreams = streams.filter((stream: Stream) => + stream.isBroadcastInCountry(country) ) - - const countrySubdivisionsCodes = countrySubdivisions.map( - (subdivision: Subdivision) => `s/${subdivision.code}` - ) - - const countryAreaCodes = regions - .filter((region: Region) => region.countries.includes(country.code)) - .map((region: Region) => `r/${region.code}`) - .concat(countrySubdivisionsCodes) - .add(`c/${country.code}`) - - const countryStreams = streams.filter(stream => - stream.broadcastArea.intersects(countryAreaCodes) - ) - if (countryStreams.isEmpty()) return const playlist = new Playlist(countryStreams, { public: true }) const filepath = `countries/${country.code.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() }) + ) - countrySubdivisions.forEach(async (subdivision: Subdivision) => { - const subdivisionStreams = streams.filter(stream => - stream.broadcastArea.includes(`s/${subdivision.code}`) + country.getSubdivisions().forEach(async (subdivision: Subdivision) => { + const subdivisionStreams = streams.filter((stream: Stream) => + stream.isBroadcastInSubdivision(subdivision) ) if (subdivisionStreams.isEmpty()) return @@ -70,16 +50,22 @@ export class CountriesGenerator implements Generator { const playlist = new Playlist(subdivisionStreams, { public: true }) const filepath = `subdivisions/${subdivision.code.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() }) + ) }) }) - const internationalStreams = streams.filter(stream => stream.isInternational()) - if (internationalStreams.notEmpty()) { - const playlist = new Playlist(internationalStreams, { public: true }) - const filepath = 'countries/int.m3u' - await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) - } + const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea()) + const undefinedPlaylist = new Playlist(undefinedStreams, { public: true }) + const undefinedFilepath = 'countries/undefined.m3u' + await this.storage.save(undefinedFilepath, undefinedPlaylist.toString()) + this.logger.info( + JSON.stringify({ + type: 'country', + filepath: undefinedFilepath, + count: undefinedPlaylist.streams.count() + }) + ) } } diff --git a/scripts/generators/indexCategoryGenerator.ts b/scripts/generators/indexCategoryGenerator.ts index 8fd5f2cbf..529ee8336 100644 --- a/scripts/generators/indexCategoryGenerator.ts +++ b/scripts/generators/indexCategoryGenerator.ts @@ -26,14 +26,14 @@ export class IndexCategoryGenerator implements Generator { let groupedStreams = new Collection() streams.forEach((stream: Stream) => { - if (stream.noCategories()) { + if (!stream.hasCategories()) { const streamClone = stream.clone() streamClone.groupTitle = 'Undefined' groupedStreams.add(streamClone) return } - stream.categories.forEach((category: Category) => { + stream.getCategories().forEach((category: Category) => { const streamClone = stream.clone() streamClone.groupTitle = category.name groupedStreams.push(streamClone) @@ -48,6 +48,6 @@ export class IndexCategoryGenerator implements Generator { const playlist = new Playlist(groupedStreams, { public: true }) const filepath = 'index.category.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/generators/indexCountryGenerator.ts b/scripts/generators/indexCountryGenerator.ts index dcdb214f8..c65a43734 100644 --- a/scripts/generators/indexCountryGenerator.ts +++ b/scripts/generators/indexCountryGenerator.ts @@ -1,29 +1,20 @@ import { Generator } from './generator' import { Collection, Storage, Logger } from '@freearhey/core' -import { Stream, Playlist, Country, Subdivision, Region } from '../models' +import { Stream, Playlist, Country } from '../models' import { PUBLIC_DIR } from '../constants' type IndexCountryGeneratorProps = { streams: Collection - regions: Collection - countries: Collection - subdivisions: Collection logger: Logger } export class IndexCountryGenerator implements Generator { streams: Collection - countries: Collection - regions: Collection - subdivisions: Collection storage: Storage logger: Logger - constructor({ streams, regions, countries, subdivisions, logger }: IndexCountryGeneratorProps) { + constructor({ streams, logger }: IndexCountryGeneratorProps) { this.streams = streams - this.countries = countries - this.regions = regions - this.subdivisions = subdivisions this.storage = new Storage(PUBLIC_DIR) this.logger = logger } @@ -32,10 +23,10 @@ export class IndexCountryGenerator implements Generator { let groupedStreams = new Collection() this.streams - .orderBy(stream => stream.getTitle()) - .filter(stream => stream.isSFW()) - .forEach(stream => { - if (stream.noBroadcastArea()) { + .orderBy((stream: Stream) => stream.getTitle()) + .filter((stream: Stream) => stream.isSFW()) + .forEach((stream: Stream) => { + if (!stream.hasBroadcastArea()) { const streamClone = stream.clone() streamClone.groupTitle = 'Undefined' groupedStreams.add(streamClone) @@ -48,7 +39,7 @@ export class IndexCountryGenerator implements Generator { groupedStreams.add(streamClone) } - this.getStreamBroadcastCountries(stream).forEach((country: Country) => { + stream.getBroadcastCountries().forEach((country: Country) => { const streamClone = stream.clone() streamClone.groupTitle = country.name groupedStreams.add(streamClone) @@ -65,40 +56,6 @@ export class IndexCountryGenerator implements Generator { const playlist = new Playlist(groupedStreams, { public: true }) const filepath = 'index.country.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) - } - - getStreamBroadcastCountries(stream: Stream) { - const groupedRegions = this.regions.keyBy((region: Region) => region.code) - const groupedCountries = this.countries.keyBy((country: Country) => country.code) - const groupedSubdivisions = this.subdivisions.keyBy( - (subdivision: Subdivision) => subdivision.code - ) - - let broadcastCountries = new Collection() - - stream.broadcastArea.forEach(broadcastAreaCode => { - const [type, code] = broadcastAreaCode.split('/') - switch (type) { - case 'c': - broadcastCountries.add(code) - break - case 'r': - if (code !== 'INT' && groupedRegions.has(code)) { - broadcastCountries = broadcastCountries.concat(groupedRegions.get(code).countries) - } - break - case 's': - if (groupedSubdivisions.has(code)) { - broadcastCountries.add(groupedSubdivisions.get(code).country) - } - break - } - }) - - return broadcastCountries - .uniq() - .map(code => groupedCountries.get(code)) - .filter(Boolean) + this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/generators/indexGenerator.ts b/scripts/generators/indexGenerator.ts index b4389ff5f..fafda061f 100644 --- a/scripts/generators/indexGenerator.ts +++ b/scripts/generators/indexGenerator.ts @@ -27,6 +27,6 @@ export class IndexGenerator implements Generator { const playlist = new Playlist(sfwStreams, { public: true }) const filepath = 'index.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/generators/indexLanguageGenerator.ts b/scripts/generators/indexLanguageGenerator.ts index a64ffaada..1116eb740 100644 --- a/scripts/generators/indexLanguageGenerator.ts +++ b/scripts/generators/indexLanguageGenerator.ts @@ -22,17 +22,17 @@ export class IndexLanguageGenerator implements Generator { async generate(): Promise { let groupedStreams = new Collection() this.streams - .orderBy(stream => stream.getTitle()) - .filter(stream => stream.isSFW()) - .forEach(stream => { - if (stream.noLanguages()) { + .orderBy((stream: Stream) => stream.getTitle()) + .filter((stream: Stream) => stream.isSFW()) + .forEach((stream: Stream) => { + if (!stream.hasLanguages()) { const streamClone = stream.clone() streamClone.groupTitle = 'Undefined' groupedStreams.add(streamClone) return } - stream.languages.forEach((language: Language) => { + stream.getLanguages().forEach((language: Language) => { const streamClone = stream.clone() streamClone.groupTitle = language.name groupedStreams.add(streamClone) @@ -47,6 +47,6 @@ export class IndexLanguageGenerator implements Generator { const playlist = new Playlist(groupedStreams, { public: true }) const filepath = 'index.language.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/generators/indexNsfwGenerator.ts b/scripts/generators/indexNsfwGenerator.ts index a1f0a8062..a89cf0a10 100644 --- a/scripts/generators/indexNsfwGenerator.ts +++ b/scripts/generators/indexNsfwGenerator.ts @@ -25,6 +25,6 @@ export class IndexNsfwGenerator implements Generator { const playlist = new Playlist(allStreams, { public: true }) const filepath = 'index.nsfw.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/generators/indexRegionGenerator.ts b/scripts/generators/indexRegionGenerator.ts index 55affcaa3..c67d29bd5 100644 --- a/scripts/generators/indexRegionGenerator.ts +++ b/scripts/generators/indexRegionGenerator.ts @@ -28,14 +28,14 @@ export class IndexRegionGenerator implements Generator { .orderBy((stream: Stream) => stream.getTitle()) .filter((stream: Stream) => stream.isSFW()) .forEach((stream: Stream) => { - if (stream.noBroadcastArea()) { + if (!stream.hasBroadcastArea()) { const streamClone = stream.clone() streamClone.groupTitle = 'Undefined' groupedStreams.push(streamClone) return } - this.getStreamRegions(stream).forEach((region: Region) => { + stream.getBroadcastRegions().forEach((region: Region) => { const streamClone = stream.clone() streamClone.groupTitle = region.name groupedStreams.push(streamClone) @@ -50,34 +50,6 @@ export class IndexRegionGenerator implements Generator { const playlist = new Playlist(groupedStreams, { public: true }) const filepath = 'index.region.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) - } - - getStreamRegions(stream: Stream) { - let streamRegions = new Collection() - stream.broadcastArea.forEach(broadcastAreaCode => { - const [type, code] = broadcastAreaCode.split('/') - switch (type) { - case 'r': - const groupedRegions = this.regions.keyBy((region: Region) => region.code) - streamRegions.add(groupedRegions.get(code)) - break - case 's': - const [countryCode] = code.split('-') - const subdivisionRegions = this.regions.filter((region: Region) => - region.countries.includes(countryCode) - ) - streamRegions = streamRegions.concat(subdivisionRegions) - break - case 'c': - const countryRegions = this.regions.filter((region: Region) => - region.countries.includes(code) - ) - streamRegions = streamRegions.concat(countryRegions) - break - } - }) - - return streamRegions + this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/generators/languagesGenerator.ts b/scripts/generators/languagesGenerator.ts index d40d53d9b..114fcddb2 100644 --- a/scripts/generators/languagesGenerator.ts +++ b/scripts/generators/languagesGenerator.ts @@ -18,35 +18,40 @@ export class LanguagesGenerator implements Generator { async generate(): Promise { const streams = this.streams - .orderBy(stream => stream.getTitle()) - .filter(stream => stream.isSFW()) + .orderBy((stream: Stream) => stream.getTitle()) + .filter((stream: Stream) => stream.isSFW()) let languages = new Collection() streams.forEach((stream: Stream) => { - languages = languages.concat(stream.languages) + languages = languages.concat(stream.getLanguages()) }) languages + .filter(Boolean) .uniqBy((language: Language) => language.code) .orderBy((language: Language) => language.name) .forEach(async (language: Language) => { - const languageStreams = streams.filter(stream => stream.hasLanguage(language)) + const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language)) if (languageStreams.isEmpty()) return const playlist = new Playlist(languageStreams, { public: true }) const filepath = `languages/${language.code}.m3u` await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + ) }) - const undefinedStreams = streams.filter(stream => stream.noLanguages()) + const undefinedStreams = streams.filter((stream: Stream) => !stream.hasLanguages()) if (undefinedStreams.isEmpty()) return const playlist = new Playlist(undefinedStreams, { public: true }) const filepath = 'languages/undefined.m3u' await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + ) } } diff --git a/scripts/generators/regionsGenerator.ts b/scripts/generators/regionsGenerator.ts index 9c29ee3a2..fb0a5d688 100644 --- a/scripts/generators/regionsGenerator.ts +++ b/scripts/generators/regionsGenerator.ts @@ -1,53 +1,61 @@ import { Generator } from './generator' import { Collection, Storage, Logger } from '@freearhey/core' -import { Playlist, Subdivision, Region } from '../models' +import { Playlist, Region, Stream } from '../models' import { PUBLIC_DIR } from '../constants' type RegionsGeneratorProps = { streams: Collection regions: Collection - subdivisions: Collection logger: Logger } export class RegionsGenerator implements Generator { streams: Collection regions: Collection - subdivisions: Collection storage: Storage logger: Logger - constructor({ streams, regions, subdivisions, logger }: RegionsGeneratorProps) { + constructor({ streams, regions, logger }: RegionsGeneratorProps) { this.streams = streams this.regions = regions - this.subdivisions = subdivisions this.storage = new Storage(PUBLIC_DIR) this.logger = logger } async generate(): Promise { const streams = this.streams - .orderBy(stream => stream.getTitle()) - .filter(stream => stream.isSFW()) + .orderBy((stream: Stream) => stream.getTitle()) + .filter((stream: Stream) => stream.isSFW()) this.regions.forEach(async (region: Region) => { - if (region.code === 'INT') return + if (region.isWorldwide()) return - const regionSubdivisionsCodes = this.subdivisions - .filter((subdivision: Subdivision) => region.countries.indexOf(subdivision.country) > -1) - .map((subdivision: Subdivision) => `s/${subdivision.code}`) - - const regionCodes = region.countries - .map((code: string) => `c/${code}`) - .concat(regionSubdivisionsCodes) - .add(`r/${region.code}`) - - const regionStreams = streams.filter(stream => stream.broadcastArea.intersects(regionCodes)) + const regionStreams = streams.filter((stream: Stream) => stream.isBroadcastInRegion(region)) const playlist = new Playlist(regionStreams, { public: true }) const filepath = `regions/${region.code.toLowerCase()}.m3u` await this.storage.save(filepath, playlist.toString()) - this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() })) + this.logger.info( + JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }) + ) }) + + const internationalStreams = streams.filter((stream: Stream) => stream.isInternational()) + const internationalPlaylist = new Playlist(internationalStreams, { public: true }) + const internationalFilepath = 'regions/int.m3u' + await this.storage.save(internationalFilepath, internationalPlaylist.toString()) + this.logger.info( + JSON.stringify({ + type: 'region', + filepath: internationalFilepath, + count: internationalPlaylist.streams.count() + }) + ) + + const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea()) + const playlist = new Playlist(undefinedStreams, { public: true }) + const filepath = 'regions/undefined.m3u' + await this.storage.save(filepath, playlist.toString()) + this.logger.info(JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() })) } } diff --git a/scripts/models/blocked.ts b/scripts/models/blocked.ts index 1bc38886b..29041278b 100644 --- a/scripts/models/blocked.ts +++ b/scripts/models/blocked.ts @@ -5,13 +5,13 @@ type BlockedProps = { } export class Blocked { - channel: string + channelId: string reason: string ref: string - constructor({ ref, reason, channel }: BlockedProps) { - this.channel = channel - this.reason = reason - this.ref = ref + constructor(data: BlockedProps) { + this.channelId = data.channel + this.reason = data.reason + this.ref = data.ref } } diff --git a/scripts/models/broadcastArea.ts b/scripts/models/broadcastArea.ts new file mode 100644 index 000000000..2b96b7f91 --- /dev/null +++ b/scripts/models/broadcastArea.ts @@ -0,0 +1,11 @@ +type BroadcastAreaProps = { + code: string +} + +export class BroadcastArea { + code: string + + constructor(data: BroadcastAreaProps) { + this.code = data.code + } +} diff --git a/scripts/models/category.ts b/scripts/models/category.ts index 885cea849..17ff9af12 100644 --- a/scripts/models/category.ts +++ b/scripts/models/category.ts @@ -1,4 +1,4 @@ -type CategoryProps = { +type CategoryData = { id: string name: string } @@ -7,8 +7,8 @@ export class Category { id: string name: string - constructor({ id, name }: CategoryProps) { - this.id = id - this.name = name + constructor(data: CategoryData) { + this.id = data.id + this.name = data.name } } diff --git a/scripts/models/channel.ts b/scripts/models/channel.ts index dd7a7a1d9..1d4c5cf8d 100644 --- a/scripts/models/channel.ts +++ b/scripts/models/channel.ts @@ -1,17 +1,16 @@ -import { Collection } from '@freearhey/core' +import { Collection, Dictionary } from '@freearhey/core' +import { Category, Country, Subdivision } from './index' -type ChannelProps = { +type ChannelData = { id: string name: string alt_names: string[] network: string - owners: string[] + owners: Collection country: string subdivision: string city: string - broadcast_area: string[] - languages: string[] - categories: string[] + categories: Collection is_nsfw: boolean launched: string closed: string @@ -24,56 +23,86 @@ export class Channel { id: string name: string altNames: Collection - network: string + network?: string owners: Collection - country: string - subdivision: string - city: string - broadcastArea: Collection - languages: Collection - categories: Collection + countryCode: string + country?: Country + subdivisionCode?: string + subdivision?: Subdivision + cityName?: string + categoryIds: Collection + categories?: Collection isNSFW: boolean - launched: string - closed: string - replacedBy: string - website: string + launched?: string + closed?: string + replacedBy?: string + website?: string logo: string - constructor({ - id, - name, - alt_names, - network, - owners, - country, - subdivision, - city, - broadcast_area, - languages, - categories, - is_nsfw, - launched, - closed, - replaced_by, - website, - logo - }: ChannelProps) { - this.id = id - this.name = name - this.altNames = new Collection(alt_names) - this.network = network - this.owners = new Collection(owners) - this.country = country - this.subdivision = subdivision - this.city = city - this.broadcastArea = new Collection(broadcast_area) - this.languages = new Collection(languages) - this.categories = new Collection(categories) - this.isNSFW = is_nsfw - this.launched = launched - this.closed = closed - this.replacedBy = replaced_by - this.website = website - this.logo = logo + constructor(data: ChannelData) { + this.id = data.id + this.name = data.name + this.altNames = new Collection(data.alt_names) + this.network = data.network || undefined + this.owners = new Collection(data.owners) + this.countryCode = data.country + this.subdivisionCode = data.subdivision || undefined + this.cityName = data.city || undefined + this.categoryIds = new Collection(data.categories) + this.isNSFW = data.is_nsfw + this.launched = data.launched || undefined + this.closed = data.closed || undefined + this.replacedBy = data.replaced_by || undefined + this.website = data.website || undefined + this.logo = data.logo + } + + withSubdivision(subdivisionsGroupedByCode: Dictionary): this { + if (!this.subdivisionCode) return this + + this.subdivision = subdivisionsGroupedByCode.get(this.subdivisionCode) + + return this + } + + withCountry(countriesGroupedByCode: Dictionary): this { + this.country = countriesGroupedByCode.get(this.countryCode) + + return this + } + + withCategories(groupedCategories: Dictionary): this { + this.categories = this.categoryIds + .map((id: string) => groupedCategories.get(id)) + .filter(Boolean) + + return this + } + + getCountry(): Country | undefined { + return this.country + } + + getSubdivision(): Subdivision | undefined { + return this.subdivision + } + + getCategories(): Collection { + return this.categories || new Collection() + } + + hasCategories(): boolean { + return !!this.categories && this.categories.notEmpty() + } + + hasCategory(category: Category): boolean { + return ( + !!this.categories && + this.categories.includes((_category: Category) => _category.id === category.id) + ) + } + + isSFW(): boolean { + return this.isNSFW === false } } diff --git a/scripts/models/country.ts b/scripts/models/country.ts index 5b33858ce..ac822a235 100644 --- a/scripts/models/country.ts +++ b/scripts/models/country.ts @@ -1,20 +1,58 @@ -type CountryProps = { +import { Collection, Dictionary } from '@freearhey/core' +import { Region, Language } from '.' + +type CountryData = { code: string name: string - languages: string[] + lang: string flag: string } export class Country { code: string name: string - languages: string[] flag: string + languageCode: string + language?: Language + subdivisions?: Collection + regions?: Collection + + constructor(data: CountryData) { + this.code = data.code + this.name = data.name + this.flag = data.flag + this.languageCode = data.lang + } + + withSubdivisions(subdivisionsGroupedByCountryCode: Dictionary): this { + this.subdivisions = subdivisionsGroupedByCountryCode.get(this.code) || new Collection() + + return this + } + + withRegions(regions: Collection): this { + this.regions = regions.filter( + (region: Region) => region.code !== 'INT' && region.includesCountryCode(this.code) + ) + + return this + } + + withLanguage(languagesGroupedByCode: Dictionary): this { + this.language = languagesGroupedByCode.get(this.languageCode) + + return this + } + + getLanguage(): Language | undefined { + return this.language + } + + getRegions(): Collection { + return this.regions || new Collection() + } - constructor({ code, name, languages, flag }: CountryProps) { - this.code = code - this.name = name - this.languages = languages - this.flag = flag + getSubdivisions(): Collection { + return this.subdivisions || new Collection() } } diff --git a/scripts/models/feed.ts b/scripts/models/feed.ts new file mode 100644 index 000000000..db4be5011 --- /dev/null +++ b/scripts/models/feed.ts @@ -0,0 +1,196 @@ +import { Collection, Dictionary } from '@freearhey/core' +import { Country, Language, Region, Channel, Subdivision } from './index' + +type FeedData = { + channel: string + id: string + name: string + is_main: boolean + broadcast_area: Collection + languages: Collection + timezones: Collection + video_format: string +} + +export class Feed { + channelId: string + channel?: Channel + id: string + name: string + isMain: boolean + broadcastAreaCodes: Collection + broadcastCountryCodes: Collection + broadcastCountries?: Collection + broadcastRegionCodes: Collection + broadcastRegions?: Collection + broadcastSubdivisionCodes: Collection + broadcastSubdivisions?: Collection + languageCodes: Collection + languages?: Collection + timezoneIds: Collection + timezones?: Collection + videoFormat: string + + constructor(data: FeedData) { + this.channelId = data.channel + this.id = data.id + this.name = data.name + this.isMain = data.is_main + this.broadcastAreaCodes = new Collection(data.broadcast_area) + this.languageCodes = new Collection(data.languages) + this.timezoneIds = new Collection(data.timezones) + this.videoFormat = data.video_format + this.broadcastCountryCodes = new Collection() + this.broadcastRegionCodes = new Collection() + this.broadcastSubdivisionCodes = new Collection() + + this.broadcastAreaCodes.forEach((areaCode: string) => { + const [type, code] = areaCode.split('/') + + switch (type) { + case 'c': + this.broadcastCountryCodes.add(code) + break + case 'r': + this.broadcastRegionCodes.add(code) + break + case 's': + this.broadcastSubdivisionCodes.add(code) + break + } + }) + } + + withChannel(channelsGroupedById: Dictionary): this { + this.channel = channelsGroupedById.get(this.channelId) + + return this + } + + withLanguages(languagesGroupedByCode: Dictionary): this { + this.languages = this.languageCodes + .map((code: string) => languagesGroupedByCode.get(code)) + .filter(Boolean) + + return this + } + + withTimezones(timezonesGroupedById: Dictionary): this { + this.timezones = this.timezoneIds + .map((id: string) => timezonesGroupedById.get(id)) + .filter(Boolean) + + return this + } + + withBroadcastSubdivisions(subdivisionsGroupedByCode: Dictionary): this { + this.broadcastSubdivisions = this.broadcastSubdivisionCodes.map((code: string) => + subdivisionsGroupedByCode.get(code) + ) + + return this + } + + withBroadcastCountries( + countriesGroupedByCode: Dictionary, + regionsGroupedByCode: Dictionary, + subdivisionsGroupedByCode: Dictionary + ): this { + let broadcastCountries = new Collection() + + if (this.isInternational()) { + this.broadcastCountries = broadcastCountries + return this + } + + this.broadcastCountryCodes.forEach((code: string) => { + broadcastCountries.add(countriesGroupedByCode.get(code)) + }) + + this.broadcastRegionCodes.forEach((code: string) => { + const region: Region = regionsGroupedByCode.get(code) + broadcastCountries = broadcastCountries.concat(region.countryCodes) + }) + + this.broadcastSubdivisionCodes.forEach((code: string) => { + const subdivision: Subdivision = subdivisionsGroupedByCode.get(code) + broadcastCountries.add(countriesGroupedByCode.get(subdivision.countryCode)) + }) + + this.broadcastCountries = broadcastCountries.uniq().filter(Boolean) + + return this + } + + withBroadcastRegions(regions: Collection, regionsGroupedByCode: Dictionary): this { + if (!this.broadcastCountries) return this + const countriesCodes = this.broadcastCountries.map((country: Country) => country.code) + + const broadcastRegions = regions.filter((region: Region) => + region.countryCodes.intersects(countriesCodes) + ) + + if (this.isInternational()) broadcastRegions.add(regionsGroupedByCode.get('INT')) + + this.broadcastRegions = broadcastRegions + + return this + } + + hasBroadcastArea(): boolean { + return ( + this.isInternational() || (!!this.broadcastCountries && this.broadcastCountries.notEmpty()) + ) + } + + getBroadcastCountries(): Collection { + return this.broadcastCountries || new Collection() + } + + getBroadcastRegions(): Collection { + return this.broadcastRegions || new Collection() + } + + getTimezones(): Collection { + return this.timezones || new Collection() + } + + getLanguages(): Collection { + return this.languages || new Collection() + } + + hasLanguages(): boolean { + return !!this.languages && this.languages.notEmpty() + } + + hasLanguage(language: Language): boolean { + return ( + !!this.languages && + this.languages.includes((_language: Language) => _language.code === language.code) + ) + } + + isInternational(): boolean { + return this.broadcastAreaCodes.includes('r/INT') + } + + isBroadcastInSubdivision(subdivision: Subdivision): boolean { + if (this.isInternational()) return false + + return this.broadcastSubdivisionCodes.includes(subdivision.code) + } + + isBroadcastInCountry(country: Country): boolean { + if (this.isInternational()) return false + + return this.getBroadcastCountries().includes( + (_country: Country) => _country.code === country.code + ) + } + + isBroadcastInRegion(region: Region): boolean { + if (this.isInternational()) return false + + return this.getBroadcastRegions().includes((_region: Region) => _region.code === region.code) + } +} diff --git a/scripts/models/index.ts b/scripts/models/index.ts index 9782fdae8..83a9380ed 100644 --- a/scripts/models/index.ts +++ b/scripts/models/index.ts @@ -8,3 +8,6 @@ export * from './language' export * from './country' export * from './region' export * from './subdivision' +export * from './feed' +export * from './broadcastArea' +export * from './timezone' diff --git a/scripts/models/language.ts b/scripts/models/language.ts index 84433abca..aeda5e6c2 100644 --- a/scripts/models/language.ts +++ b/scripts/models/language.ts @@ -1,4 +1,4 @@ -type LanguageProps = { +type LanguageData = { code: string name: string } @@ -7,8 +7,8 @@ export class Language { code: string name: string - constructor({ code, name }: LanguageProps) { - this.code = code - this.name = name + constructor(data: LanguageData) { + this.code = data.code + this.name = data.name } } diff --git a/scripts/models/region.ts b/scripts/models/region.ts index 72b30c192..928b48f06 100644 --- a/scripts/models/region.ts +++ b/scripts/models/region.ts @@ -1,6 +1,7 @@ -import { Collection } from '@freearhey/core' +import { Collection, Dictionary } from '@freearhey/core' +import { Subdivision } from '.' -type RegionProps = { +type RegionData = { code: string name: string countries: string[] @@ -9,11 +10,43 @@ type RegionProps = { export class Region { code: string name: string - countries: Collection + countryCodes: Collection + countries?: Collection + subdivisions?: Collection - constructor({ code, name, countries }: RegionProps) { - this.code = code - this.name = name - this.countries = new Collection(countries) + constructor(data: RegionData) { + this.code = data.code + this.name = data.name + this.countryCodes = new Collection(data.countries) + } + + withCountries(countriesGroupedByCode: Dictionary): this { + this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code)) + + return this + } + + withSubdivisions(subdivisions: Collection): this { + this.subdivisions = subdivisions.filter( + (subdivision: Subdivision) => this.countryCodes.indexOf(subdivision.countryCode) > -1 + ) + + return this + } + + getSubdivisions(): Collection { + return this.subdivisions || new Collection() + } + + getCountries(): Collection { + return this.countries || new Collection() + } + + includesCountryCode(code: string): boolean { + return this.countryCodes.includes((countryCode: string) => countryCode === code) + } + + isWorldwide(): boolean { + return this.code === 'INT' } } diff --git a/scripts/models/stream.ts b/scripts/models/stream.ts index 53d244126..aabee817f 100644 --- a/scripts/models/stream.ts +++ b/scripts/models/stream.ts @@ -1,64 +1,166 @@ -import { URL, Collection } from '@freearhey/core' -import { Category, Language } from './index' +import { URL, Collection, Dictionary } from '@freearhey/core' +import { Feed, Channel, Category, Region, Subdivision, Country, Language } from './index' +import parser from 'iptv-playlist-parser' -type StreamProps = { +export class Stream { name: string url: string - filepath: string + id?: string + groupTitle: string + channelId?: string + channel?: Channel + feedId?: string + feed?: Feed + filepath?: string line: number - channel?: string - httpReferrer?: string - httpUserAgent?: string label?: string quality?: string -} - -export class Stream { - channel: string - filepath: string - line: number - httpReferrer: string - label: string - name: string - quality: string - url: string - httpUserAgent: string - logo: string - broadcastArea: Collection - categories: Collection - languages: Collection - isNSFW: boolean - groupTitle: string + httpReferrer?: string + httpUserAgent?: string removed: boolean = false - constructor({ - channel, - filepath, - line, - httpReferrer, - label, - name, - quality, - url, - httpUserAgent - }: StreamProps) { - this.channel = channel || '' - this.filepath = filepath - this.line = line - this.httpReferrer = httpReferrer || '' - this.label = label || '' + constructor(data: parser.PlaylistItem) { + if (!data.name) throw new Error('"name" property is required') + if (!data.url) throw new Error('"url" property is required') + + const [channelId, feedId] = data.tvg.id.split('@') + const { name, label, quality } = parseTitle(data.name) + + this.id = data.tvg.id || undefined + this.feedId = feedId || undefined + this.channelId = channelId || undefined + this.line = data.line + this.label = label || undefined this.name = name - this.quality = quality || '' - this.url = url - this.httpUserAgent = httpUserAgent || '' - this.logo = '' - this.broadcastArea = new Collection() - this.categories = new Collection() - this.languages = new Collection() - this.isNSFW = false + this.quality = quality || undefined + this.url = data.url + this.httpReferrer = data.http.referrer || undefined + this.httpUserAgent = data.http['user-agent'] || undefined this.groupTitle = 'Undefined' } + withChannel(channelsGroupedById: Dictionary): this { + if (!this.channelId) return this + + this.channel = channelsGroupedById.get(this.channelId) + + return this + } + + withFeed(feedsGroupedByChannelId: Dictionary): this { + if (!this.channelId) return this + + const channelFeeds = feedsGroupedByChannelId.get(this.channelId) || [] + if (this.feedId) this.feed = channelFeeds.find((feed: Feed) => feed.id === this.feedId) + if (!this.feed) this.feed = channelFeeds.find((feed: Feed) => feed.isMain) + + return this + } + + setId(id: string): this { + this.id = id + + return this + } + + setChannelId(channelId: string): this { + this.channelId = channelId + + return this + } + + setFeedId(feedId: string | undefined): this { + this.feedId = feedId + + return this + } + + setLabel(label: string): this { + this.label = label + + return this + } + + setQuality(quality: string): this { + this.quality = quality + + return this + } + + setHttpUserAgent(httpUserAgent: string): this { + this.httpUserAgent = httpUserAgent + + return this + } + + setHttpReferrer(httpReferrer: string): this { + this.httpReferrer = httpReferrer + + return this + } + + setFilepath(filepath: string): this { + this.filepath = filepath + + return this + } + + updateFilepath(): this { + if (!this.channel) return this + + this.filepath = `${this.channel.countryCode.toLowerCase()}.m3u` + + return this + } + + getFilepath(): string { + return this.filepath || '' + } + + getHttpReferrer(): string { + return this.httpReferrer || '' + } + + getHttpUserAgent(): string { + return this.httpUserAgent || '' + } + + getQuality(): string { + return this.quality || '' + } + + hasQuality(): boolean { + return !!this.quality + } + + getHorizontalResolution(): number { + if (!this.hasQuality()) return 0 + + return parseInt(this.getQuality().replace(/p|i/, '')) + } + + updateName(): this { + if (!this.channel) return this + + this.name = this.channel.name + if (this.feed && !this.feed.isMain) { + this.name += ` ${this.feed.name}` + } + + return this + } + + updateId(): this { + if (!this.channel) return this + if (this.feed) { + this.id = `${this.channel.id}@${this.feed.id}` + } else { + this.id = this.channel.id + } + + return this + } + normalizeURL() { const url = new URL(this.url) @@ -81,36 +183,75 @@ export class Stream { return !!this.channel } - hasCategories(): boolean { - return this.categories.notEmpty() + getBroadcastRegions(): Collection { + return this.feed ? this.feed.getBroadcastRegions() : new Collection() + } + + getBroadcastCountries(): Collection { + return this.feed ? this.feed.getBroadcastCountries() : new Collection() + } + + hasBroadcastArea(): boolean { + return this.feed ? this.feed.hasBroadcastArea() : false + } + + isSFW(): boolean { + return this.channel ? this.channel.isSFW() : true } - noCategories(): boolean { - return this.categories.isEmpty() + hasCategories(): boolean { + return this.channel ? this.channel.hasCategories() : false } hasCategory(category: Category): boolean { - return this.categories.includes((_category: Category) => _category.id === category.id) + return this.channel ? this.channel.hasCategory(category) : false + } + + getCategoryNames(): string[] { + return this.getCategories() + .map((category: Category) => category.name) + .sort() + .all() + } + + getCategories(): Collection { + return this.channel ? this.channel.getCategories() : new Collection() } - noLanguages(): boolean { - return this.languages.isEmpty() + getLanguages(): Collection { + return this.feed ? this.feed.getLanguages() : new Collection() } - hasLanguage(language: Language): boolean { - return this.languages.includes((_language: Language) => _language.code === language.code) + hasLanguages() { + return this.feed ? this.feed.hasLanguages() : false } - noBroadcastArea(): boolean { - return this.broadcastArea.isEmpty() + hasLanguage(language: Language) { + return this.feed ? this.feed.hasLanguage(language) : false + } + + getBroadcastAreaCodes(): Collection { + return this.feed ? this.feed.broadcastAreaCodes : new Collection() + } + + isBroadcastInSubdivision(subdivision: Subdivision): boolean { + return this.feed ? this.feed.isBroadcastInSubdivision(subdivision) : false + } + + isBroadcastInCountry(country: Country): boolean { + return this.feed ? this.feed.isBroadcastInCountry(country) : false + } + + isBroadcastInRegion(region: Region): boolean { + return this.feed ? this.feed.isBroadcastInRegion(region) : false } isInternational(): boolean { - return this.broadcastArea.includes('r/INT') + return this.feed ? this.feed.isInternational() : false } - isSFW(): boolean { - return this.isNSFW === false + getLogo(): string { + return this?.channel?.logo || '' } getTitle(): string { @@ -127,15 +268,25 @@ export class Stream { return title } + getLabel(): string { + return this.label || '' + } + + getId(): string { + return this.id || '' + } + data() { return { + id: this.id, channel: this.channel, + feed: this.feed, filepath: this.filepath, - httpReferrer: this.httpReferrer, label: this.label, name: this.name, quality: this.quality, url: this.url, + httpReferrer: this.httpReferrer, httpUserAgent: this.httpUserAgent, line: this.line } @@ -143,7 +294,8 @@ export class Stream { toJSON() { return { - channel: this.channel || null, + channel: this.channelId || null, + feed: this.feedId || null, url: this.url, referrer: this.httpReferrer || null, user_agent: this.httpUserAgent || null @@ -151,10 +303,10 @@ export class Stream { } toString(options: { public: boolean }) { - let output = `#EXTINF:-1 tvg-id="${this.channel}"` + let output = `#EXTINF:-1 tvg-id="${this.getId()}"` if (options.public) { - output += ` tvg-logo="${this.logo}" group-title="${this.groupTitle}"` + output += ` tvg-logo="${this.getLogo()}" group-title="${this.groupTitle}"` } if (this.httpReferrer) { @@ -180,3 +332,16 @@ export class Stream { return output } } + +function parseTitle(title: string): { name: string; label: string; quality: string } { + const [, label] = title.match(/ \[(.*)\]$/) || [null, ''] + title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '') + const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, ''] + title = title.replace(new RegExp(` \\(${quality}\\)$`), '') + + return { name: title, label, quality } +} + +function escapeRegExp(text) { + return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') +} diff --git a/scripts/models/subdivision.ts b/scripts/models/subdivision.ts index c3209ca3d..d6795fea3 100644 --- a/scripts/models/subdivision.ts +++ b/scripts/models/subdivision.ts @@ -1,4 +1,7 @@ -type SubdivisionProps = { +import { Dictionary } from '@freearhey/core' +import { Country } from '.' + +type SubdivisionData = { code: string name: string country: string @@ -7,11 +10,18 @@ type SubdivisionProps = { export class Subdivision { code: string name: string - country: string + countryCode: string + country?: Country + + constructor(data: SubdivisionData) { + this.code = data.code + this.name = data.name + this.countryCode = data.country + } + + withCountry(countriesGroupedByCode: Dictionary): this { + this.country = countriesGroupedByCode.get(this.countryCode) - constructor({ code, name, country }: SubdivisionProps) { - this.code = code - this.name = name - this.country = country + return this } } diff --git a/scripts/models/timezone.ts b/scripts/models/timezone.ts new file mode 100644 index 000000000..b519f0e06 --- /dev/null +++ b/scripts/models/timezone.ts @@ -0,0 +1,30 @@ +import { Collection, Dictionary } from '@freearhey/core' + +type TimezoneData = { + id: string + utc_offset: string + countries: string[] +} + +export class Timezone { + id: string + utcOffset: string + countryCodes: Collection + countries?: Collection + + constructor(data: TimezoneData) { + this.id = data.id + this.utcOffset = data.utc_offset + this.countryCodes = new Collection(data.countries) + } + + withCountries(countriesGroupedByCode: Dictionary): this { + this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code)) + + return this + } + + getCountries(): Collection { + return this.countries || new Collection() + } +} diff --git a/scripts/tables/categoryTable.ts b/scripts/tables/categoryTable.ts index a3fb49f14..f82f3ffd4 100644 --- a/scripts/tables/categoryTable.ts +++ b/scripts/tables/categoryTable.ts @@ -11,6 +11,7 @@ export class CategoryTable implements Table { const dataStorage = new Storage(DATA_DIR) const categoriesContent = await dataStorage.json('categories.json') const categories = new Collection(categoriesContent).map(data => new Category(data)) + const categoriesGroupedById = categories.keyBy((category: Category) => category.id) const parser = new LogParser() const logsStorage = new Storage(LOGS_DIR) @@ -19,13 +20,12 @@ export class CategoryTable implements Table { let data = new Collection() parser .parse(generatorsLog) - .filter((logItem: LogItem) => logItem.filepath.includes('categories/')) + .filter((logItem: LogItem) => logItem.type === 'category') .forEach((logItem: LogItem) => { const file = new File(logItem.filepath) const categoryId = file.name() - const category: Category = categories.first( - (category: Category) => category.id === categoryId - ) + const category: Category = categoriesGroupedById.get(categoryId) + data.add([ category ? category.name : 'ZZ', category ? category.name : 'Undefined', diff --git a/scripts/tables/countryTable.ts b/scripts/tables/countryTable.ts index 06a72490e..2f50ccedf 100644 --- a/scripts/tables/countryTable.ts +++ b/scripts/tables/countryTable.ts @@ -12,34 +12,31 @@ export class CountryTable implements Table { const countriesContent = await dataStorage.json('countries.json') const countries = new Collection(countriesContent).map(data => new Country(data)) - + const countriesGroupedByCode = countries.keyBy((country: Country) => country.code) const subdivisionsContent = await dataStorage.json('subdivisions.json') const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data)) + const subdivisionsGroupedByCode = subdivisions.keyBy( + (subdivision: Subdivision) => subdivision.code + ) const parser = new LogParser() const logsStorage = new Storage(LOGS_DIR) const generatorsLog = await logsStorage.load('generators.log') + const parsed = parser.parse(generatorsLog) let data = new Collection() - parser - .parse(generatorsLog) - .filter( - (logItem: LogItem) => - logItem.filepath.includes('countries/') || logItem.filepath.includes('subdivisions/') - ) + + parsed + .filter((logItem: LogItem) => logItem.type === 'subdivision') .forEach((logItem: LogItem) => { const file = new File(logItem.filepath) const code = file.name().toUpperCase() const [countryCode, subdivisionCode] = code.split('-') || ['', ''] + const country = countriesGroupedByCode.get(countryCode) - if (subdivisionCode) { - const subdivision = subdivisions.first( - (subdivision: Subdivision) => subdivision.code === code - ) + if (country && subdivisionCode) { + const subdivision = subdivisionsGroupedByCode.get(code) if (subdivision) { - const country = countries.first( - (country: Country) => country.code === subdivision.country - ) data.add([ `${country.name}/${subdivision.name}`, `      ${subdivision.name}`, @@ -47,18 +44,28 @@ export class CountryTable implements Table { `https://iptv-org.github.io/iptv/${logItem.filepath}` ]) } - } else if (countryCode === 'INT') { + } + }) + + parsed + .filter((logItem: LogItem) => logItem.type === 'country') + .forEach((logItem: LogItem) => { + const file = new File(logItem.filepath) + const code = file.name().toUpperCase() + const [countryCode] = code.split('-') || ['', ''] + const country = countriesGroupedByCode.get(countryCode) + + if (country) { data.add([ - 'ZZ', - '🌍 International', + country.name, + `${country.flag} ${country.name}`, logItem.count, `https://iptv-org.github.io/iptv/${logItem.filepath}` ]) } else { - const country = countries.first((country: Country) => country.code === countryCode) data.add([ - country.name, - `${country.flag} ${country.name}`, + 'ZZ', + 'Undefined', logItem.count, `https://iptv-org.github.io/iptv/${logItem.filepath}` ]) diff --git a/scripts/tables/languageTable.ts b/scripts/tables/languageTable.ts index f0b54d242..2014ba676 100644 --- a/scripts/tables/languageTable.ts +++ b/scripts/tables/languageTable.ts @@ -11,6 +11,7 @@ export class LanguageTable implements Table { const dataStorage = new Storage(DATA_DIR) const languagesContent = await dataStorage.json('languages.json') const languages = new Collection(languagesContent).map(data => new Language(data)) + const languagesGroupedByCode = languages.keyBy((language: Language) => language.code) const parser = new LogParser() const logsStorage = new Storage(LOGS_DIR) @@ -19,13 +20,11 @@ export class LanguageTable implements Table { let data = new Collection() parser .parse(generatorsLog) - .filter((logItem: LogItem) => logItem.filepath.includes('languages/')) + .filter((logItem: LogItem) => logItem.type === 'language') .forEach((logItem: LogItem) => { const file = new File(logItem.filepath) const languageCode = file.name() - const language: Language = languages.first( - (language: Language) => language.code === languageCode - ) + const language: Language = languagesGroupedByCode.get(languageCode) data.add([ language ? language.name : 'ZZ', diff --git a/scripts/tables/regionTable.ts b/scripts/tables/regionTable.ts index 60a6e5ff7..84eeaaa4a 100644 --- a/scripts/tables/regionTable.ts +++ b/scripts/tables/regionTable.ts @@ -11,6 +11,7 @@ export class RegionTable implements Table { const dataStorage = new Storage(DATA_DIR) const regionsContent = await dataStorage.json('regions.json') const regions = new Collection(regionsContent).map(data => new Region(data)) + const regionsGroupedByCode = regions.keyBy((region: Region) => region.code) const parser = new LogParser() const logsStorage = new Storage(LOGS_DIR) @@ -19,22 +20,35 @@ export class RegionTable implements Table { let data = new Collection() parser .parse(generatorsLog) - .filter((logItem: LogItem) => logItem.filepath.includes('regions/')) + .filter((logItem: LogItem) => logItem.type === 'region') .forEach((logItem: LogItem) => { const file = new File(logItem.filepath) const regionCode = file.name().toUpperCase() - const region: Region = regions.first((region: Region) => region.code === regionCode) + const region: Region = regionsGroupedByCode.get(regionCode) if (region) { data.add([ region.name, + region.name, + logItem.count, + `https://iptv-org.github.io/iptv/${logItem.filepath}` + ]) + } else { + data.add([ + 'ZZZ', + 'Undefined', logItem.count, `https://iptv-org.github.io/iptv/${logItem.filepath}` ]) } }) - data = data.orderBy(item => item[0]) + data = data + .orderBy(item => item[0]) + .map(item => { + item.shift() + return item + }) const table = new HTMLTable(data.all(), [ { name: 'Region', align: 'left' },