Merge pull request #20708 from iptv-org/patch-2025.03.3

Patch 2025.03.3

CONTRIBUTING.md

@@ -137,6 +137,7 @@ To run scripts use the `npm run <script-name>` command.
 - `playlist:generate`: generates all public playlists.
 - `playlist:validate`: сhecks ids and links in internal playlists for errors.
 - `playlist:lint`: сhecks internal playlists for syntax errors.
+- `playlist:test`: tests links in internal playlists.
 - `playlist:deploy`: allows to manually publish all generated via `playlist:generate` playlists. To run the script you must provide your [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) with write access to the repository.
 - `readme:update`: updates the list of playlists in [README.md](README.md).
 - `report:create`: creates a report on current issues.

package.json

@@ -12,6 +12,7 @@
     "playlist:generate": "tsx scripts/commands/playlist/generate.ts",
     "playlist:validate": "tsx scripts/commands/playlist/validate.ts",
     "playlist:lint": "npx m3u-linter -c m3u-linter.json",
+    "playlist:test": "tsx scripts/commands/playlist/test.ts",
     "playlist:deploy": "npx gh-pages-clean && npx gh-pages -m \"Deploy to GitHub Pages\" -d .gh-pages -r https://$GITHUB_TOKEN@github.com/iptv-org/iptv.git",
     "readme:update": "tsx scripts/commands/readme/update.ts",
     "report:create": "tsx scripts/commands/report/create.ts",
@@ -39,8 +40,8 @@
     "@eslint/eslintrc": "^3.3.0",
     "@eslint/js": "^9.21.0",
     "@freearhey/core": "^0.2.1",
-    "@octokit/core": "^4.2.1",
+    "@octokit/core": "^6.1.4",
-    "@octokit/plugin-paginate-rest": "^7.1.2",
+    "@octokit/plugin-paginate-rest": "^11.4.3",
     "@octokit/plugin-rest-endpoint-methods": "^7.1.3",
     "@octokit/types": "^11.1.0",
     "@types/cli-progress": "^3.11.3",
@@ -49,12 +50,16 @@
     "@types/numeral": "^2.0.3",
     "@typescript-eslint/eslint-plugin": "^8.18.1",
     "@typescript-eslint/parser": "^8.18.1",
+    "async-es": "^3.2.6",
     "axios": "^1.7.9",
     "chalk": "^4.1.2",
     "cli-progress": "^3.12.0",
+    "command-exists": "^1.2.9",
     "commander": "^8.3.0",
+    "console-table-printer": "^2.12.1",
     "eslint": "^9.17.0",
     "globals": "^16.0.0",
+    "iptv-checker": "^0.29.1",
     "iptv-playlist-parser": "^0.13.0",
     "jest-expect-message": "^1.1.3",
     "lodash": "^4.17.21",

scripts/commands/playlist/test.ts

@@ -0,0 +1,166 @@
+import { Logger, Storage, Collection } from '@freearhey/core'
+import { ROOT_DIR, STREAMS_DIR } from '../../constants'
+import { PlaylistParser, StreamTester, CliTable } from '../../core'
+import { Stream } from '../../models'
+import { program } from 'commander'
+import { eachLimit } from 'async-es'
+import commandExists from 'command-exists'
+import chalk from 'chalk'
+import os from 'node:os'
+import dns from 'node:dns'
+
+const cpus = os.cpus()
+
+const LIVE_UPDATE_INTERVAL = 5000
+const LIVE_UPDATE_MAX_STREAMS = 100
+
+let errors = 0
+let warnings = 0
+let results = {}
+let interval
+let streams = new Collection()
+let isLiveUpdateEnabled = true
+
+program
+  .argument('[filepath]', 'Path to file to validate')
+  .option(
+    '-p, --parallel <number>',
+    'Batch size of streams to test concurrently',
+    cpus.length,
+    (value: string) => parseInt(value)
+  )
+  .option('-x, --proxy <url>', 'Use the specified proxy')
+  .parse(process.argv)
+
+const options = program.opts()
+
+const logger = new Logger()
+const tester = new StreamTester()
+
+async function main() {
+  const storage = new Storage(ROOT_DIR)
+
+  if (await isOffline()) {
+    logger.error(chalk.red('Internet connection is required for the script to work'))
+
+    return
+  }
+
+  if (!commandExists.sync('ffprobe')) {
+    logger.error(
+      chalk.red(
+        'For the script to work, the “ffprobe” library must be installed (https://ffmpeg.org/download.html)'
+      )
+    )
+
+    return
+  }
+
+  logger.info('loading streams...')
+  const parser = new PlaylistParser({ storage })
+  const files = program.args.length ? program.args : await storage.list(`${STREAMS_DIR}/*.m3u`)
+  streams = await parser.parse(files)
+
+  logger.info(`found ${streams.count()} streams`)
+  if (streams.count() > LIVE_UPDATE_MAX_STREAMS) isLiveUpdateEnabled = false
+
+  logger.info('starting...')
+  if (!isLiveUpdateEnabled) {
+    drawTable()
+    interval = setInterval(() => {
+      drawTable()
+    }, LIVE_UPDATE_INTERVAL)
+  }
+
+  await eachLimit(
+    streams.all(),
+    options.parallel,
+    async (stream: Stream) => {
+      await runTest(stream)
+
+      if (isLiveUpdateEnabled) {
+        drawTable()
+      }
+    },
+    onFinish
+  )
+}
+
+main()
+
+async function runTest(stream: Stream) {
+  const key = stream.filepath + stream.channel + stream.url
+  results[key] = chalk.white('LOADING...')
+
+  const result = await tester.test(stream)
+
+  let status = ''
+  const errorStatusCodes = ['HTTP_NOT_FOUND']
+  if (result.status.ok) status = chalk.green('OK')
+  else if (errorStatusCodes.includes(result.status.code)) {
+    status = chalk.red(result.status.code)
+    errors++
+  } else {
+    status = chalk.yellow(result.status.code)
+    warnings++
+  }
+
+  results[key] = status
+}
+
+function drawTable() {
+  process.stdout.write('\u001b[3J\u001b[1J')
+  console.clear()
+
+  const streamsGrouped = streams.groupBy((stream: Stream) => stream.filepath)
+  for (const filepath of streamsGrouped.keys()) {
+    const streams: Stream[] = streamsGrouped.get(filepath)
+
+    const table = new CliTable({
+      columns: [
+        { name: '', alignment: 'center', minLen: 3, maxLen: 3 },
+        { name: 'tvg-id', alignment: 'left', color: 'green', minLen: 25, maxLen: 25 },
+        { name: 'url', alignment: 'left', color: 'green', minLen: 100, maxLen: 100 },
+        { name: 'status', alignment: 'left', minLen: 25, maxLen: 25 }
+      ]
+    })
+    streams.forEach((stream: Stream, index: number) => {
+      const status = results[stream.filepath + stream.channel + stream.url] || chalk.gray('PENDING')
+
+      const row = {
+        '': index,
+        'tvg-id': stream.channel.length > 25 ? stream.channel.slice(0, 22) + '...' : stream.channel,
+        url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url,
+        status
+      }
+      table.append(row)
+    })
+
+    process.stdout.write(`\n${chalk.underline(filepath)}\n`)
+
+    process.stdout.write(table.toString())
+  }
+}
+
+function onFinish() {
+  clearInterval(interval)
+
+  drawTable()
+
+  logger.error(`\n${errors + warnings} problems (${errors} errors, ${warnings} warnings)`)
+
+  if (errors > 0) {
+    process.exit(1)
+  }
+
+  process.exit(0)
+}
+
+async function isOffline() {
+  return new Promise((resolve, reject) => {
+    dns.lookup('info.cern.ch', err => {
+      if (err) resolve(true)
+      reject(false)
+    })
+  }).catch(() => {})
+}

scripts/core/cliTable.ts

@@ -0,0 +1,21 @@
+import { Table } from 'console-table-printer'
+
+export class CliTable {
+  table: Table
+
+  constructor(options?) {
+    this.table = new Table(options)
+  }
+
+  append(row) {
+    this.table.addRow(row)
+  }
+
+  render() {
+    this.table.printTable()
+  }
+
+  toString() {
+    return this.table.render()
+  }
+}

scripts/core/index.ts

@@ -7,3 +7,5 @@ export * from './issueParser'
 export * from './htmlTable'
 export * from './apiClient'
 export * from './issueData'
+export * from './streamTester'
+export * from './cliTable'

scripts/core/playlistParser.ts

@@ -1,8 +1,6 @@
 import { Collection, Storage } from '@freearhey/core'
 import parser from 'iptv-playlist-parser'
 import { Stream } from '../models'
-import path from 'path'
-import { STREAMS_DIR } from '../constants'
 
 export class PlaylistParser {
   storage: Storage
@@ -15,8 +13,7 @@ export class PlaylistParser {
     let streams = new Collection()
 
     for (const filepath of files) {
-      const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
+      const _streams: Collection = await this.parseFile(filepath)
-      const _streams: Collection = await this.parseFile(relativeFilepath)
       streams = streams.concat(_streams)
     }
 

scripts/core/streamTester.ts

@@ -0,0 +1,27 @@
+import { Stream } from '../models'
+import { IPTVChecker } from 'iptv-checker'
+import { TESTING } from '../constants'
+
+export class StreamTester {
+  checker: IPTVChecker
+
+  constructor() {
+    this.checker = new IPTVChecker()
+  }
+
+  async test(stream: Stream) {
+    if (TESTING) {
+      const results = (await import('../../tests/__data__/input/test_results/all.js')).default
+
+      return results[stream.url]
+    } else {
+      return this.checker.checkStream({
+        url: stream.url,
+        http: {
+          referrer: stream.httpReferrer,
+          'user-agent': stream.httpUserAgent
+        }
+      })
+    }
+  }
+}

tests/__data__/input/streams_test/ag.m3u

@@ -0,0 +1,5 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="ABSTV.ag",ABS TV
+https://tego-cdn2a.sibercdn.com/Live_TV-ABSTV-10/tracks-v3a1/rewind-7200.m3u8?token=e5f61e7be8363eb781b4bdfe591bf917dd529c1a-SjY3NzRTbDZQNnFQVkZaNkZja2RxV3JKc1VBa05zQkdMNStJakRGV0VTTzNrOEVGVUlIQmxta1NLV0o3bzdVdQ-1736094545-1736008145
+#EXTINF:-1 tvg-id="ABSTV.ag",ABS TV (1080p) [Not 24/7]
+https://query-streamlink.herokuapp.com/iptv-query?streaming-ip=https://www.twitch.tv/absliveantigua3

tests/__data__/input/test_results/all.js

@@ -0,0 +1,14 @@
+module.exports = {
+  'https://query-streamlink.herokuapp.com/iptv-query?streaming-ip=https://www.twitch.tv/absliveantigua3':
+    {
+      url: 'https://query-streamlink.herokuapp.com/iptv-query?streaming-ip=https://www.twitch.tv/absliveantigua3',
+      http: { referrer: '', 'user-agent': '' },
+      status: { ok: false, code: 'HTTP_NOT_FOUND', message: 'HTTP 404 Not Found' }
+    },
+  'https://tego-cdn2a.sibercdn.com/Live_TV-ABSTV-10/tracks-v3a1/rewind-7200.m3u8?token=e5f61e7be8363eb781b4bdfe591bf917dd529c1a-SjY3NzRTbDZQNnFQVkZaNkZja2RxV3JKc1VBa05zQkdMNStJakRGV0VTTzNrOEVGVUlIQmxta1NLV0o3bzdVdQ-1736094545-1736008145':
+    {
+      url: 'https://tego-cdn2a.sibercdn.com/Live_TV-ABSTV-10/tracks-v3a1/rewind-7200.m3u8?token=e5f61e7be8363eb781b4bdfe591bf917dd529c1a-SjY3NzRTbDZQNnFQVkZaNkZja2RxV3JKc1VBa05zQkdMNStJakRGV0VTTzNrOEVGVUlIQmxta1NLV0o3bzdVdQ-1736094545-1736008145',
+      http: { referrer: '', 'user-agent': '' },
+      status: { ok: false, code: 'HTTP_FORBIDDEN', message: 'HTTP 403 Forbidden' }
+    }
+}

tests/commands/playlist/test.test.ts

@@ -0,0 +1,19 @@
+import { execSync } from 'child_process'
+
+type ExecError = {
+  status: number
+  stdout: string
+}
+
+it('shows an error if the playlist contains a broken link', () => {
+  try {
+    execSync('ROOT_DIR=tests/__data__/input npm run playlist:test streams_test/ag.m3u', {
+      encoding: 'utf8'
+    })
+    process.exit(1)
+  } catch (error) {
+    expect((error as ExecError).status).toBe(1)
+    expect((error as ExecError).stdout).toContain('streams_test/ag.m3u')
+    expect((error as ExecError).stdout).toContain('2 problems (1 errors, 1 warnings)')
+  }
+})