diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 000000000..ad1bd9022 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,62 @@ +name: format +on: + workflow_dispatch: + # pull_request: + # types: + # - closed +jobs: + main: + # if: ${{ github.event.pull_request.merged == true }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: getsentry/action-github-app-token@v2 + if: ${{ !env.ACT }} + id: create-app-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: actions/checkout@v3 + if: ${{ !env.ACT }} + with: + fetch-depth: 2 + token: ${{ steps.create-app-token.outputs.token }} + - name: setup git + run: | + git config user.name "iptv-bot[bot]" + git config user.email "84861620+iptv-bot[bot]@users.noreply.github.com" + - uses: tj-actions/changed-files@v35 + id: files + with: + files: streams/*.m3u + - name: download data from api + run: | + mkdir -p temp/data + curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json + curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json + - uses: actions/setup-node@v3 + if: ${{ !env.ACT }} + with: + node-version: 18 + cache: 'npm' + - name: install dependencies + run: npm install + - name: format internal playlists + run: npm run playlist:format + - name: check internal playlists + run: | + npm run playlist:lint + npm run playlist:validate + - run: git status + - name: commit changes to /streams + run: | + git add streams + git status + git commit -m "[Bot] Format /streams" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [format](https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}) workflow." --no-verify + - name: push all changes to the repository + if: ${{ !env.ACT && github.ref == 'refs/heads/master' }} + run: git push diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 3e13b6b69..dcf6bd86c 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -33,8 +33,6 @@ jobs: run: npm install - name: load api data run: npm run api:load - - name: setup database - run: npm run db:create - name: update internal playlists run: npm run playlist:update --silent >> $GITHUB_OUTPUT id: playlist-update diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2950166c..a83661b00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,21 +123,22 @@ For scripts to work, you must have [Node.js](https://nodejs.org/en) installed on To run scripts use the `npm run ` command. - `act:check`: allows to run the [check](https://github.com/iptv-org/iptv/blob/master/.github/workflows/check.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act). +- `act:format`: allows to test the [format](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act). - `act:update`: allows to test the [update](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act). - `api:load`: downloads the latest channel and stream data from the [iptv-org/api](https://github.com/iptv-org/api). - `api:generate`: generates a JSON file with all streams for the [iptv-org/api](https://github.com/iptv-org/api) repository. - `api:deploy`: allows to manually upload a JSON file created via `api:generate` to the [iptv-org/api](https://github.com/iptv-org/api) repository. To run the script you must provide your [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) with write access to the repository. -- `db:create`: сreates a temporary file `temp/database/streams.db` containing all links from the [/streams]() folder. -- `playlist:update`: triggers an update of internal playlists. The process involves processing approved requests from issues, URL normalization, and sorting links by channel name, quality, and label. +- `playlist:format`: formats internal playlists. The process includes [URL normalization](https://en.wikipedia.org/wiki/URI_normalization), duplicate removal, removing invalid id's and sorting links by channel name, quality, and label. +- `playlist:update`: triggers an update of internal playlists. The process involves processing approved requests from issues. +- `playlist:generate`: generates all public playlists. - `playlist:validate`: сhecks ids and links in internal playlists for errors. - `playlist:lint`: сhecks internal playlists for syntax errors. -- `playlist:generate`: generates all public playlists. - `playlist:deploy`: allows to manually publish all generated via `playlist:generate` playlists. To run the script you must provide your [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) with write access to the repository. - `readme:update`: updates the list of playlists in [README.md](README.md). - `report:create`: shows a list of all current requests and their status. -- `format`: (shorthand) sequentially runs the `db:create` and `db:create` commands. +- `format`: (shorthand) sequentially runs the `api:load` and `playlist:format` commands. - `check`: (shorthand) sequentially runs the `api:load`, `playlist:lint` and `playlist:validate` commands. -- `update`: (shorthand) sequentially runs the `api:load`, `db:create`, `playlist:generate`, `api:generate` and `readme:update` commands. +- `update`: (shorthand) sequentially runs the `api:load`, `playlist:generate`, `api:generate` and `readme:update` commands. - `deploy`: (shorthand) sequentially runs the `playlist:deploy` and `api:deploy` commands. - `report`: (shorthand) sequentially runs the `api:load` and `report:create` commands. - `test`: runs a test of all the scripts described above. @@ -148,5 +149,6 @@ To automate the run of the scripts described above, we use the [GitHub Actions w Each workflow includes its own set of scripts that can be run either manually or in response to an event. -- `check`: sequentially runs the `playlist:check` and `playlist:validate` scripts when a new pull request appears, and blocks the merge if it detects an error in it. -- `update`: every day at 0:00 UTC sequentially runs `api:load`, `db:create`, `playlist:update`, `playlist:lint`, `playlist:validate`, `playlist:generate`, `api:generate` and `readme:update` scripts and deploys the output files if successful. +- `check`: sequentially runs the `api:load`, `playlist:check` and `playlist:validate` scripts when a new pull request appears, and blocks the merge if it detects an error in it. +- `format`: sequentially runs `api:load`, `playlist:format`, `playlist:lint` and `playlist:validate` scripts. +- `update`: every day at 0:00 UTC sequentially runs `api:load`, `playlist:update`, `playlist:lint`, `playlist:validate`, `playlist:generate`, `api:generate` and `readme:update` scripts and deploys the output files if successful. diff --git a/act.json b/act.json new file mode 100644 index 000000000..7801daa89 --- /dev/null +++ b/act.json @@ -0,0 +1,5 @@ +{ + "pull_request": { + "merged": true + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 798b38d0d..e16a1924b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "@octokit/plugin-paginate-rest": "^7.1.2", "@octokit/plugin-rest-endpoint-methods": "^7.1.3", "@octokit/types": "^11.1.0", - "@seald-io/nedb": "^4.0.2", "@types/fs-extra": "^11.0.1", "@types/glob": "^8.1.0", "@types/jest": "^29.5.4", @@ -1338,21 +1337,6 @@ "node": ">=14" } }, - "node_modules/@seald-io/binary-search-tree": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.3.tgz", - "integrity": "sha512-qv3jnwoakeax2razYaMsGI/luWdliBLHTdC6jU55hQt1hcFqzauH/HsBollQ7IR4ySTtYhT+xyHoijpA16C+tA==" - }, - "node_modules/@seald-io/nedb": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@seald-io/nedb/-/nedb-4.0.2.tgz", - "integrity": "sha512-gJ91fT1sgh2cLXYVcTSh7khZ8LdemI8+SojCdpZ5wy+DUQ4fSrEwGqOwbdV49NDs2BBO6GeBpSb8CnhG2IW1rw==", - "dependencies": { - "@seald-io/binary-search-tree": "^1.0.3", - "localforage": "^1.9.0", - "util": "^0.12.4" - } - }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -1658,17 +1642,6 @@ "node": ">=0.10.0" } }, - "node_modules/available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/babel-jest": { "version": "29.6.4", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz", @@ -1855,18 +1828,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "peer": true }, - "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -2318,14 +2279,6 @@ "node": ">=8" } }, - "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dependencies": { - "is-callable": "^1.1.3" - } - }, "node_modules/foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -2395,7 +2348,8 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "peer": true }, "node_modules/gensync": { "version": "1.0.0-beta.2", @@ -2414,20 +2368,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/get-intrinsic": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", - "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", - "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -2595,17 +2535,6 @@ "node": ">=0.10.0" } }, - "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/graceful-fs": { "version": "4.2.9", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", @@ -2615,6 +2544,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "peer": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -2630,42 +2560,6 @@ "node": ">=8" } }, - "node_modules/has-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dependencies": { - "has-symbols": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -2681,11 +2575,6 @@ "node": ">=10.17.0" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" - }, "node_modules/import-local": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", @@ -2737,32 +2626,6 @@ "validator": "^13.7.0" } }, - "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", @@ -2800,20 +2663,6 @@ "node": ">=6" } }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-glob": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", @@ -2864,20 +2713,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "dependencies": { - "which-typed-array": "^1.1.11" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-valid-path": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz", @@ -3808,14 +3643,6 @@ "node": ">=6" } }, - "node_modules/lie": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz", - "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=", - "dependencies": { - "immediate": "~3.0.5" - } - }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -3856,14 +3683,6 @@ "node": ">=4" } }, - "node_modules/localforage": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz", - "integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==", - "dependencies": { - "lie": "3.1.1" - } - }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -5117,18 +4936,6 @@ "node": ">= 4.0.0" } }, - "node_modules/util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -5191,24 +4998,6 @@ "node": ">= 8" } }, - "node_modules/which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -6319,21 +6108,6 @@ "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", "optional": true }, - "@seald-io/binary-search-tree": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.3.tgz", - "integrity": "sha512-qv3jnwoakeax2razYaMsGI/luWdliBLHTdC6jU55hQt1hcFqzauH/HsBollQ7IR4ySTtYhT+xyHoijpA16C+tA==" - }, - "@seald-io/nedb": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@seald-io/nedb/-/nedb-4.0.2.tgz", - "integrity": "sha512-gJ91fT1sgh2cLXYVcTSh7khZ8LdemI8+SojCdpZ5wy+DUQ4fSrEwGqOwbdV49NDs2BBO6GeBpSb8CnhG2IW1rw==", - "requires": { - "@seald-io/binary-search-tree": "^1.0.3", - "localforage": "^1.9.0", - "util": "^0.12.4" - } - }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -6608,11 +6382,6 @@ "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==" }, - "available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" - }, "babel-jest": { "version": "29.6.4", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz", @@ -6761,15 +6530,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "peer": true }, - "call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "requires": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - } - }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -7089,14 +6849,6 @@ "path-exists": "^4.0.0" } }, - "for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "requires": { - "is-callable": "^1.1.3" - } - }, "foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -7145,7 +6897,8 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "peer": true }, "gensync": { "version": "1.0.0-beta.2", @@ -7158,17 +6911,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "get-intrinsic": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", - "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", - "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3" - } - }, "get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -7293,14 +7035,6 @@ } } }, - "gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "requires": { - "get-intrinsic": "^1.1.3" - } - }, "graceful-fs": { "version": "4.2.9", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", @@ -7310,6 +7044,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "peer": true, "requires": { "function-bind": "^1.1.1" } @@ -7319,24 +7054,6 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" }, - "has-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" - }, - "has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" - }, - "has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "requires": { - "has-symbols": "^1.0.2" - } - }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -7349,11 +7066,6 @@ "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "peer": true }, - "immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" - }, "import-local": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", @@ -7393,20 +7105,6 @@ "validator": "^13.7.0" } }, - "is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "requires": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - } - }, - "is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" - }, "is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", @@ -7432,14 +7130,6 @@ "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "peer": true }, - "is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "requires": { - "has-tostringtag": "^1.0.0" - } - }, "is-glob": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", @@ -7472,14 +7162,6 @@ "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "peer": true }, - "is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "requires": { - "which-typed-array": "^1.1.11" - } - }, "is-valid-path": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz", @@ -8186,14 +7868,6 @@ "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", "peer": true }, - "lie": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz", - "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=", - "requires": { - "immediate": "~3.0.5" - } - }, "lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -8227,14 +7901,6 @@ } } }, - "localforage": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz", - "integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==", - "requires": { - "lie": "3.1.1" - } - }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -9116,18 +8782,6 @@ "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" }, - "util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "requires": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -9180,18 +8834,6 @@ "isexe": "^2.0.0" } }, - "which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "requires": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - } - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 24762d030..55ec7891f 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,12 @@ "name": "iptv", "scripts": { "act:check": "act pull_request -W .github/workflows/check.yml", + "act:format": "act workflow_dispatch -W .github/workflows/format.yml", "act:update": "act workflow_dispatch -W .github/workflows/update.yml", "api:load": "./scripts/commands/api/load.sh", "api:generate": "npm run ts-node scripts/commands/api/generate.ts", "api:deploy": "npx gh-pages-clean && npx gh-pages -a -m \"Deploy to iptv-org/api\" -d .api -r https://$GITHUB_TOKEN@github.com/iptv-org/api.git", - "db:create": "npm run ts-node scripts/commands/database/create.ts", + "playlist:format": "npm run ts-node scripts/commands/playlist/format.ts", "playlist:update": "npm run ts-node scripts/commands/playlist/update.ts", "playlist:generate": "npm run ts-node scripts/commands/playlist/generate.ts", "playlist:validate": "npm run ts-node scripts/commands/playlist/validate.ts", @@ -14,9 +15,9 @@ "playlist:deploy": "npx gh-pages-clean && npx gh-pages -m \"Deploy to GitHub Pages\" -d .gh-pages -r https://$GITHUB_TOKEN@github.com/iptv-org/iptv.git", "readme:update": "npm run ts-node scripts/commands/readme/update.ts", "report:create": "npm run ts-node scripts/commands/report/create.ts", - "format": "npm run db:create && npm run playlist:format", + "format": "npm run api:load && npm run playlist:format", "check": "npm run api:load && npm run playlist:lint && npm run playlist:validate", - "update": "npm run api:load && npm run db:create && npm run playlist:generate && npm run api:generate && npm run readme:update", + "update": "npm run api:load && npm run playlist:generate && npm run api:generate && npm run readme:update", "deploy": "npm run playlist:deploy && npm run api:deploy", "report": "npm run api:load && npm run report:create", "test": "jest --runInBand", @@ -45,7 +46,6 @@ "@octokit/plugin-paginate-rest": "^7.1.2", "@octokit/plugin-rest-endpoint-methods": "^7.1.3", "@octokit/types": "^11.1.0", - "@seald-io/nedb": "^4.0.2", "@types/fs-extra": "^11.0.1", "@types/glob": "^8.1.0", "@types/jest": "^29.5.4", diff --git a/scripts/commands/api/generate.ts b/scripts/commands/api/generate.ts index edb08db48..0c0b033d7 100644 --- a/scripts/commands/api/generate.ts +++ b/scripts/commands/api/generate.ts @@ -1,16 +1,16 @@ -import { API_DIR, DB_DIR } from '../../constants' -import { Logger, Database, Collection, Storage } from '../../core' +import { API_DIR, STREAMS_DIR } from '../../constants' +import { Logger, PlaylistParser, Storage } from '../../core' import { Stream } from '../../models' async function main() { const logger = new Logger() - logger.info(`loading streams...`) - const db = new Database(DB_DIR) - const dbStreams = await db.load('streams.db') - const docs = await dbStreams.find({}) - - const streams = new Collection(docs as any[]) + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ storage: streamsStorage }) + const files = await streamsStorage.list('**/*.m3u') + let streams = await parser.parse(files) + streams = streams .map(data => new Stream(data)) .orderBy((stream: Stream) => stream.channel) .map((stream: Stream) => stream.toJSON()) @@ -18,8 +18,8 @@ async function main() { logger.info(`found ${streams.count()} streams`) logger.info('saving to .api/streams.json...') - const storage = new Storage(API_DIR) - await storage.save('streams.json', streams.toJSON()) + const apiStorage = new Storage(API_DIR) + await apiStorage.save('streams.json', streams.toJSON()) } main() diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh index 866074753..7dea082ea 100755 --- a/scripts/commands/api/load.sh +++ b/scripts/commands/api/load.sh @@ -4,7 +4,6 @@ mkdir -p temp/data curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json curl -L -o temp/data/categories.json https://iptv-org.github.io/api/categories.json curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json -curl -L -o temp/data/streams.json https://iptv-org.github.io/api/streams.json curl -L -o temp/data/countries.json https://iptv-org.github.io/api/countries.json curl -L -o temp/data/languages.json https://iptv-org.github.io/api/languages.json curl -L -o temp/data/regions.json https://iptv-org.github.io/api/regions.json diff --git a/scripts/commands/database/create.ts b/scripts/commands/database/create.ts deleted file mode 100644 index fee17a1f7..000000000 --- a/scripts/commands/database/create.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Storage, Logger, PlaylistParser, Collection, Database } from '../../core' -import { Stream, Playlist } from '../../models' -import { STREAMS_DIR, DB_DIR } from '../../constants' - -async function main() { - const logger = new Logger() - - logger.info(`looking for streams...`) - const storage = new Storage(STREAMS_DIR) - const parser = new PlaylistParser({ - storage - }) - const files = await storage.list(`**/*.m3u`) - let streams = new Collection() - for (let filepath of files) { - const playlist: Playlist = await parser.parse(filepath) - streams = streams.concat(playlist.streams) - } - - logger.info(`found ${streams.count()} streams`) - - logger.info('clean up the storage...') - const dbStorage = new Storage(DB_DIR) - await dbStorage.clear('streams.db') - - logger.info('saving streams to the database...') - const db = new Database(DB_DIR) - const dbStreams = await db.load('streams.db') - const data = streams.map((stream: Stream) => stream.data()).all() - await dbStreams.insert(data) -} - -main() diff --git a/scripts/commands/playlist/format.ts b/scripts/commands/playlist/format.ts new file mode 100644 index 000000000..862b90e36 --- /dev/null +++ b/scripts/commands/playlist/format.ts @@ -0,0 +1,67 @@ +import { STREAMS_DIR, DATA_DIR } from '../../constants' +import { Storage, Logger, PlaylistParser, Collection } from '../../core' +import { Stream, Playlist, Channel } from '../../models' +import { program } from 'commander' + +program.argument('[filepath]', 'Path to file to validate').parse(process.argv) + +async function main() { + const storage = new Storage(STREAMS_DIR) + const logger = new Logger() + + logger.info('loading channels from api...') + const dataStorage = new Storage(DATA_DIR) + const channelsContent = await dataStorage.json('channels.json') + const groupedChannels = new Collection(channelsContent) + .map(data => new Channel(data)) + .keyBy((channel: Channel) => channel.id) + + logger.info('loading streams...') + const parser = new PlaylistParser({ storage }) + const files = program.args.length ? program.args : await storage.list('**/*.m3u') + let streams = await parser.parse(files) + + logger.info(`found ${streams.count()} streams`) + + logger.info('normalizing links...') + streams = streams.map(stream => { + stream.normalizeURL() + return stream + }) + + logger.info('removing duplicates...') + streams = streams.uniqBy(stream => stream.url) + + logger.info('removing wrong id...') + streams = streams.map((stream: Stream) => { + if (groupedChannels.missing(stream.channel)) { + stream.channel = '' + } + + return stream + }) + + logger.info('sorting links...') + streams = streams.orderBy( + [ + (stream: Stream) => stream.name, + (stream: Stream) => parseInt(stream.quality.replace('p', '')), + (stream: Stream) => stream.label, + (stream: Stream) => stream.url + ], + ['asc', 'desc', 'asc', 'asc'] + ) + + logger.info('saving...') + const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath) + for (let filepath of groupedStreams.keys()) { + const streams = groupedStreams.get(filepath) || [] + + if (!streams.length) return + + const playlist = new Playlist(streams, { public: false }) + await storage.save(filepath, playlist.toString()) + } +} + +main() diff --git a/scripts/commands/playlist/generate.ts b/scripts/commands/playlist/generate.ts index b21c44be8..44a75be23 100644 --- a/scripts/commands/playlist/generate.ts +++ b/scripts/commands/playlist/generate.ts @@ -1,6 +1,5 @@ -import { File, Storage } from '../../core' +import { File, PlaylistParser, Storage } from '../../core' import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models' -import { Database } from '../../core/database' import { Collection } from '../../core/collection' import { Logger } from '../../core/logger' import _ from 'lodash' @@ -16,32 +15,31 @@ import { IndexLanguageGenerator, IndexRegionGenerator } from '../../generators' -import { DATA_DIR, DB_DIR, LOGS_DIR } from '../../constants' +import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants' async function main() { const logger = new Logger() + const dataStorage = new Storage(DATA_DIR) - const storage = new Storage(DATA_DIR) - - const channelsContent = await storage.json('channels.json') + logger.info('loading data from api...') + const channelsContent = await dataStorage.json('channels.json') const channels = new Collection(channelsContent).map(data => new Channel(data)) - - const categoriesContent = await storage.json('categories.json') + const categoriesContent = await dataStorage.json('categories.json') const categories = new Collection(categoriesContent).map(data => new Category(data)) - - const countriesContent = await storage.json('countries.json') + const countriesContent = await dataStorage.json('countries.json') const countries = new Collection(countriesContent).map(data => new Country(data)) - - const languagesContent = await storage.json('languages.json') + const languagesContent = await dataStorage.json('languages.json') const languages = new Collection(languagesContent).map(data => new Language(data)) - - const regionsContent = await storage.json('regions.json') + const regionsContent = await dataStorage.json('regions.json') const regions = new Collection(regionsContent).map(data => new Region(data)) - - const subdivisionsContent = await storage.json('subdivisions.json') + const subdivisionsContent = await dataStorage.json('subdivisions.json') const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data)) - const streams = await loadStreams({ channels, categories, languages }) + logger.info('loading streams...') + let streams = await loadStreams({ channels, categories, languages }) + let totalStreams = streams.count() + streams = streams.uniqBy((stream: Stream) => stream.channel || _.uniqueId()) + logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`) const generatorsLogger = new Logger({ stream: await new Storage(LOGS_DIR).createStream(`generators.log`) @@ -49,7 +47,6 @@ async function main() { logger.info('generating categories/...') await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate() - logger.info('generating countries/...') await new CountriesGenerator({ countries, @@ -58,10 +55,8 @@ async function main() { subdivisions, logger: generatorsLogger }).generate() - logger.info('generating languages/...') await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate() - logger.info('generating regions/...') await new RegionsGenerator({ streams, @@ -69,16 +64,12 @@ async function main() { subdivisions, logger: generatorsLogger }).generate() - logger.info('generating index.m3u...') await new IndexGenerator({ streams, logger: generatorsLogger }).generate() - logger.info('generating index.nsfw.m3u...') await new IndexNsfwGenerator({ streams, logger: generatorsLogger }).generate() - logger.info('generating index.category.m3u...') await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate() - logger.info('generating index.country.m3u...') await new IndexCountryGenerator({ streams, @@ -87,10 +78,8 @@ async function main() { subdivisions, logger: generatorsLogger }).generate() - logger.info('generating index.language.m3u...') await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate() - logger.info('generating index.region.m3u...') await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate() } @@ -110,13 +99,13 @@ async function loadStreams({ const groupedCategories = categories.keyBy(category => category.id) const groupedLanguages = languages.keyBy(language => language.code) - const db = new Database(DB_DIR) - const dbStreams = await db.load('streams.db') - const docs = await dbStreams.find({}) - const streams = new Collection(docs as any[]) - .map((data: any) => new Stream(data)) + const storage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ storage }) + const files = await storage.list('**/*.m3u') + let streams = await parser.parse(files) + + streams = streams .orderBy([(stream: Stream) => stream.channel, (stream: Stream) => stream.url], ['asc', 'asc']) - .uniqBy((stream: Stream) => stream.channel || _.uniqueId()) .map((stream: Stream) => { const channel: Channel | undefined = groupedChannels.get(stream.channel) diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index 723d19d46..839de24fd 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -1,6 +1,6 @@ -import { DB_DIR, DATA_DIR, STREAMS_DIR } from '../../constants' -import { Database, Storage, Logger, Collection, Dictionary, IssueLoader } from '../../core' -import { Stream, Playlist, Channel } from '../../models' +import { DATA_DIR, STREAMS_DIR } from '../../constants' +import { Storage, Logger, Collection, Dictionary, IssueLoader, PlaylistParser } from '../../core' +import { Stream, Playlist, Channel, Issue } from '../../models' let processedIssues = new Collection() let streams: Collection @@ -10,19 +10,19 @@ async function main() { const logger = new Logger({ disabled: true }) const loader = new IssueLoader() - logger.info('loading streams...') - const db = new Database(DB_DIR) - const docs = await db.load('streams.db') - const dbStreams = await docs.find({}) - - streams = new Collection(dbStreams as any[]).map(data => new Stream(data)) - - const storage = new Storage(DATA_DIR) - const channelsContent = await storage.json('channels.json') + logger.info('loading channels from api...') + const dataStorage = new Storage(DATA_DIR) + const channelsContent = await dataStorage.json('channels.json') groupedChannels = new Collection(channelsContent) .map(data => new Channel(data)) .keyBy((channel: Channel) => channel.id) + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ storage: streamsStorage }) + const files = await streamsStorage.list('**/*.m3u') + streams = await parser.parse(files) + logger.info('removing broken streams...') await removeStreams(loader) @@ -32,25 +32,7 @@ async function main() { logger.info('add new streams...') await addStreams(loader) - logger.info('normalizing links...') - streams = streams.map(stream => { - stream.normalizeURL() - return stream - }) - - logger.info('sorting links...') - streams = streams.orderBy( - [ - (stream: Stream) => stream.name, - (stream: Stream) => parseInt(stream.quality.replace('p', '')), - (stream: Stream) => stream.label, - (stream: Stream) => stream.url - ], - ['asc', 'desc', 'asc', 'asc'] - ) - logger.info('saving...') - const streamsStorage = new Storage(STREAMS_DIR) const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath) for (let filepath of groupedStreams.keys()) { const streams = groupedStreams.get(filepath) || [] @@ -69,19 +51,22 @@ main() async function removeStreams(loader: IssueLoader) { const issues = await loader.load({ labels: ['streams:remove', 'approved'] }) - issues.forEach((data: Dictionary) => { + issues.forEach((issue: Issue) => { + const data = issue.data if (data.missing('stream_url')) return const removed = streams.remove((_stream: Stream) => _stream.url === data.get('stream_url')) if (removed.notEmpty()) { - processedIssues.add(data.get('issue_number')) + processedIssues.add(issue.number) } }) } async function editStreams(loader: IssueLoader) { const issues = await loader.load({ labels: ['streams:edit', 'approved'] }) - issues.forEach((data: Dictionary) => { + issues.forEach((issue: Issue) => { + const data = issue.data + if (data.missing('stream_url')) return let stream = streams.first( @@ -111,13 +96,14 @@ async function editStreams(loader: IssueLoader) { streams.remove((_stream: Stream) => _stream.channel === stream.channel) streams.add(stream) - processedIssues.add(data.get('issue_number')) + processedIssues.add(issue.number) }) } async function addStreams(loader: IssueLoader) { const issues = await loader.load({ labels: ['streams:add', 'approved'] }) - issues.forEach((data: Dictionary) => { + issues.forEach((issue: Issue) => { + const data = issue.data if (data.missing('channel_id') || data.missing('stream_url')) return if (streams.includes((_stream: Stream) => _stream.url === data.get('stream_url'))) return @@ -138,6 +124,6 @@ async function addStreams(loader: IssueLoader) { }) streams.add(stream) - processedIssues.add(data.get('issue_number')) + processedIssues.add(issue.number) }) } diff --git a/scripts/commands/playlist/validate.ts b/scripts/commands/playlist/validate.ts index 8c6f020e3..03295a47c 100644 --- a/scripts/commands/playlist/validate.ts +++ b/scripts/commands/playlist/validate.ts @@ -5,7 +5,6 @@ import chalk from 'chalk' import { transliterate } from 'transliteration' import _ from 'lodash' import { DATA_DIR, STREAMS_DIR } from '../../constants' -import path from 'path' program.argument('[filepath]', 'Path to file to validate').parse(process.argv) @@ -19,73 +18,70 @@ async function main() { const logger = new Logger() logger.info(`loading blocklist...`) - const storage = new Storage(DATA_DIR) - const channelsContent = await storage.json('channels.json') + const dataStorage = new Storage(DATA_DIR) + const channelsContent = await dataStorage.json('channels.json') const channels = new Collection(channelsContent).map(data => new Channel(data)) - const blocklistContent = await storage.json('blocklist.json') + const blocklistContent = await dataStorage.json('blocklist.json') const blocklist = new Collection(blocklistContent).map(data => new Blocked(data)) logger.info(`found ${blocklist.count()} records`) - let errors = new Collection() - let warnings = new Collection() + logger.info('loading streams...') const streamsStorage = new Storage(STREAMS_DIR) const parser = new PlaylistParser({ storage: streamsStorage }) const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u') - for (const filepath of files) { - const file = new File(filepath) - if (file.extension() !== 'm3u') continue + const streams = await parser.parse(files) + logger.info(`found ${streams.count()} streams`) + + let errors = new Collection() + let warnings = new Collection() + let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath) + for (const filepath of groupedStreams.keys()) { + const streams = groupedStreams.get(filepath) + if (!streams) continue + + const file = new File(filepath) const [, countryCode] = file.basename().match(/([a-z]{2})(|_.*)\.m3u/i) || [null, ''] const log = new Collection() const buffer = new Dictionary() - try { - const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '') - const playlist = await parser.parse(relativeFilepath) - playlist.streams.forEach((stream: Stream) => { - const channelNotInDatabase = - stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel) - if (channelNotInDatabase) { - log.add({ - type: 'warning', - line: stream.line, - message: `"${stream.channel}" is not in the database` - }) - } - - const alreadyOnPlaylist = stream.url && buffer.has(stream.url) - if (alreadyOnPlaylist) { - log.add({ - type: 'warning', - line: stream.line, - message: `"${stream.url}" is already on the playlist` - }) - } else { - buffer.set(stream.url, true) - } - - const channelId = generateChannelId(stream.name, countryCode) - const blocked = blocklist.first( - blocked => - stream.channel.toLowerCase() === blocked.channel.toLowerCase() || - channelId.toLowerCase() === blocked.channel.toLowerCase() - ) - if (blocked) { - log.add({ - type: 'error', - line: stream.line, - message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})` - }) - } - }) - } catch (error) { - log.add({ - type: 'error', - line: 0, - message: error.message.toLowerCase() - }) - } + streams.forEach((stream: Stream) => { + const channelNotInDatabase = + stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel) + if (channelNotInDatabase) { + log.add({ + type: 'warning', + line: stream.line, + message: `"${stream.channel}" is not in the database` + }) + } + + const alreadyOnPlaylist = stream.url && buffer.has(stream.url) + if (alreadyOnPlaylist) { + log.add({ + type: 'warning', + line: stream.line, + message: `"${stream.url}" is already on the playlist` + }) + } else { + buffer.set(stream.url, true) + } + + const channelId = generateChannelId(stream.name, countryCode) + const blocked = blocklist.first( + blocked => + stream.channel.toLowerCase() === blocked.channel.toLowerCase() || + channelId.toLowerCase() === blocked.channel.toLowerCase() + ) + if (blocked) { + log.add({ + type: 'error', + line: stream.line, + message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})` + }) + } + }) if (log.notEmpty()) { logger.info(`\n${chalk.underline(filepath)}`) diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index 6194449fd..d3da0a9d4 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -1,36 +1,43 @@ -import { DATA_DIR } from '../../constants' -import { Collection, Dictionary, IssueLoader, Storage } from '../../core' -import { Blocked, Channel, Stream } from '../../models' +import { DATA_DIR, STREAMS_DIR } from '../../constants' +import { Collection, Dictionary, IssueLoader, Storage, Logger, PlaylistParser } from '../../core' +import { Blocked, Channel, Issue, Stream } from '../../models' async function main() { + const logger = new Logger() const loader = new IssueLoader() const storage = new Storage(DATA_DIR) + logger.info('loading channels from api...') const channelsContent = await storage.json('channels.json') const groupedChannels = new Collection(channelsContent) .map(data => new Channel(data)) .groupBy((channel: Channel) => channel.id) - const streamsContent = await storage.json('streams.json') - const groupedStreams = new Collection(streamsContent) - .map(data => new Stream(data)) - .groupBy((stream: Stream) => stream.url) - + logger.info('loading blocklist from api...') const blocklistContent = await storage.json('blocklist.json') const groupedBlocklist = new Collection(blocklistContent) .map(data => new Blocked(data)) .groupBy((blocked: Blocked) => blocked.channel) + logger.info('loading streams...') + const streamsStorage = new Storage(STREAMS_DIR) + const parser = new PlaylistParser({ storage: streamsStorage }) + const files = await streamsStorage.list('**/*.m3u') + const streams = await parser.parse(files) + const groupedStreams = streams.groupBy((stream: Stream) => stream.url) + + logger.info('loading issue from github...') const issues = await loader.load({ labels: ['streams:add'] }) + logger.info('creating report...') const buffer = new Dictionary() - const report = issues.map(data => { - const channelId = data.get('channel_id') || undefined - const streamUrl = data.get('stream_url') || undefined + const report = issues.map((issue: Issue) => { + const channelId = issue.data.get('channel_id') || undefined + const streamUrl = issue.data.get('stream_url') || undefined const result = new Dictionary({ - issueNumber: data.get('issue_number'), + issueNumber: issue.number, channelId, status: undefined }) diff --git a/scripts/constants.ts b/scripts/constants.ts index 7609d051f..23db757e9 100644 --- a/scripts/constants.ts +++ b/scripts/constants.ts @@ -5,7 +5,6 @@ export const README_DIR = process.env.README_DIR || './.readme' export const API_DIR = process.env.API_DIR || './.api' export const DATA_DIR = process.env.DATA_DIR || './temp/data' export const LOGS_DIR = process.env.LOGS_DIR || './temp/logs' -export const DB_DIR = process.env.DB_DIR || './temp/database' export const TESTING = process.env.NODE_ENV === 'test' ? true : false export const OWNER = 'iptv-org' export const REPO = 'iptv' diff --git a/scripts/core/database.ts b/scripts/core/database.ts deleted file mode 100644 index c2d231320..000000000 --- a/scripts/core/database.ts +++ /dev/null @@ -1,22 +0,0 @@ -import Datastore from '@seald-io/nedb' -import * as path from 'path' - -export class Database { - rootDir: string - - constructor(rootDir: string) { - this.rootDir = rootDir - } - - async load(filepath: string) { - const absFilepath = path.join(this.rootDir, filepath) - - return new Datastore({ - filename: path.resolve(absFilepath), - autoload: true, - onload: (error: Error): any => { - if (error) console.error(error.message) - } - }) - } -} diff --git a/scripts/core/index.ts b/scripts/core/index.ts index c090ad9b0..75090eca2 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -1,4 +1,3 @@ -export * from './database' export * from './logger' export * from './playlistParser' export * from './numberParser' diff --git a/scripts/core/issueParser.ts b/scripts/core/issueParser.ts index c97a7a7f2..e3fa2c779 100644 --- a/scripts/core/issueParser.ts +++ b/scripts/core/issueParser.ts @@ -1,33 +1,31 @@ import { Dictionary } from './' +import { Issue } from '../models' +import _ from 'lodash' + +const FIELDS = new Dictionary({ + 'Channel ID': 'channel_id', + 'Channel ID (required)': 'channel_id', + 'Broken Link': 'stream_url', + 'Stream URL': 'stream_url', + 'Stream URL (optional)': 'stream_url', + 'Stream URL (required)': 'stream_url', + Label: 'label', + Quality: 'quality', + 'Channel Name': 'channel_name', + 'HTTP User-Agent': 'user_agent', + 'HTTP Referrer': 'http_referrer', + Reason: 'reason', + 'What happened to the stream?': 'reason', + 'Possible Replacement (optional)': 'possible_replacement', + Notes: 'notes', + 'Notes (optional)': 'notes' +}) export class IssueParser { - parse(issue: any): Dictionary { - const data = new Dictionary() - data.set('issue_number', issue.number) - - const idDict = new Dictionary({ - 'Channel ID': 'channel_id', - 'Channel ID (required)': 'channel_id', - 'Broken Link': 'stream_url', - 'Stream URL': 'stream_url', - 'Stream URL (optional)': 'stream_url', - 'Stream URL (required)': 'stream_url', - Label: 'label', - Quality: 'quality', - 'Channel Name': 'channel_name', - 'HTTP User-Agent': 'user_agent', - 'HTTP Referrer': 'http_referrer', - Reason: 'reason', - 'What happened to the stream?': 'reason', - 'Possible Replacement (optional)': 'possible_replacement', - Notes: 'notes', - 'Notes (optional)': 'notes' - }) - + parse(issue: any): Issue { const fields = issue.body.split('###') - if (!fields.length) return data - + const data = new Dictionary() fields.forEach((field: string) => { let [_label, , _value] = field.split(/\r?\n/) _label = _label ? _label.trim() : '' @@ -35,7 +33,7 @@ export class IssueParser { if (!_label || !_value) return data - const id: string = idDict.get(_label) + const id: string = FIELDS.get(_label) const value: string = _value === '_No response_' || _value === 'None' ? '' : _value if (!id) return @@ -43,6 +41,6 @@ export class IssueParser { data.set(id, value) }) - return data + return new Issue({ number: issue.number, data }) } } diff --git a/scripts/core/playlistParser.ts b/scripts/core/playlistParser.ts index 96ab8b2b3..7a96dacc7 100644 --- a/scripts/core/playlistParser.ts +++ b/scripts/core/playlistParser.ts @@ -1,6 +1,8 @@ import parser from 'iptv-playlist-parser' -import { Playlist, Stream } from '../models' +import { Stream } from '../models' import { Collection, Storage } from './' +import path from 'path' +import { STREAMS_DIR } from '../constants' export class PlaylistParser { storage: Storage @@ -9,7 +11,19 @@ export class PlaylistParser { this.storage = storage } - async parse(filepath: string): Promise { + async parse(files: string[]): Promise { + let streams = new Collection() + + for (let filepath of files) { + const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '') + const _streams: Collection = await this.parseFile(relativeFilepath) + streams = streams.concat(_streams) + } + + return streams + } + + async parseFile(filepath: string): Promise { const streams = new Collection() const content = await this.storage.read(filepath) @@ -32,7 +46,7 @@ export class PlaylistParser { streams.add(stream) }) - return new Playlist(streams) + return streams } } diff --git a/scripts/core/storage.ts b/scripts/core/storage.ts index 128ae3ba4..81f2ede4f 100644 --- a/scripts/core/storage.ts +++ b/scripts/core/storage.ts @@ -10,10 +10,12 @@ export class Storage { this.rootDir = path.normalize(rootDir || './') } - list(pattern: string): Promise { - return glob(pattern, { + async list(pattern: string): Promise { + const files = await glob(pattern, { cwd: this.rootDir }) + + return files.sort() } async createDir(dir: string): Promise { diff --git a/scripts/models/index.ts b/scripts/models/index.ts index 9cdfbba30..9782fdae8 100644 --- a/scripts/models/index.ts +++ b/scripts/models/index.ts @@ -1,3 +1,4 @@ +export * from './issue' export * from './playlist' export * from './blocked' export * from './stream' diff --git a/scripts/models/issue.ts b/scripts/models/issue.ts new file mode 100644 index 000000000..85261261c --- /dev/null +++ b/scripts/models/issue.ts @@ -0,0 +1,16 @@ +import { Dictionary } from '../core' + +type IssueProps = { + number: number + data: Dictionary +} + +export class Issue { + number: number + data: Dictionary + + constructor({ number, data }: IssueProps) { + this.number = number + this.data = data + } +} diff --git a/tests/__data__/expected/.api/streams.json b/tests/__data__/expected/.api/streams.json index 5d7934d81..df10e542b 100644 --- a/tests/__data__/expected/.api/streams.json +++ b/tests/__data__/expected/.api/streams.json @@ -1 +1 @@ -[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null},{"channel":"BBCNewsHD.uk","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null}] \ No newline at end of file +[{"channel":"","url":"http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"},{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8","http_referrer":null,"user_agent":null},{"channel":"MeteoMedia.ca","url":"http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"VisitXTV.nl","url":"https://stream.visit-x.tv/vxtv/ngrp:live_all/30fps.m3u8","http_referrer":null,"user_agent":null},{"channel":"Zoo.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/zoo","http_referrer":null,"user_agent":null}] \ No newline at end of file diff --git a/tests/__data__/expected/.gh-pages/countries/uk.m3u b/tests/__data__/expected/.gh-pages/countries/uk.m3u deleted file mode 100644 index 5da897947..000000000 --- a/tests/__data__/expected/.gh-pages/countries/uk.m3u +++ /dev/null @@ -1,5 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 diff --git a/tests/__data__/expected/.gh-pages/index.country.m3u b/tests/__data__/expected/.gh-pages/index.country.m3u index f2aa6bde0..63f4c8e0f 100644 --- a/tests/__data__/expected/.gh-pages/index.country.m3u +++ b/tests/__data__/expected/.gh-pages/index.country.m3u @@ -9,9 +9,9 @@ http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8 http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8 #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Russia",ЛДПР ТВ (1080p) http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8 -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="United Kingdom" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] +#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="International",BBC News HD +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8 +#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] #EXTVLCOPT:http-referrer=http://imn.iq #EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 -#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="International",BBC News HD -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8 diff --git a/tests/__data__/expected/.gh-pages/index.region.m3u b/tests/__data__/expected/.gh-pages/index.region.m3u index 2d4eb8a5e..83ba6d4e4 100644 --- a/tests/__data__/expected/.gh-pages/index.region.m3u +++ b/tests/__data__/expected/.gh-pages/index.region.m3u @@ -9,20 +9,12 @@ http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8 http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8 #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Commonwealth of Independent States",ЛДПР ТВ (1080p) http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8 -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Europe" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Europe",ATV https://iptv-all.lanesh4d0w.repl.co/andorra/atv #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Europe",Zoo (720p) https://iptv-all.lanesh4d0w.repl.co/andorra/zoo #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Europe",ЛДПР ТВ (1080p) http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8 -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Europe, the Middle East and Africa" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Europe, the Middle East and Africa",ATV https://iptv-all.lanesh4d0w.repl.co/andorra/atv #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Europe, the Middle East and Africa",Zoo (720p) @@ -35,10 +27,6 @@ http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8 http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8 #EXTINF:-1 tvg-id="" tvg-logo="" group-title="South Asia",Daawah TV http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8 -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Worldwide" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Worldwide",ATV https://iptv-all.lanesh4d0w.repl.co/andorra/atv #EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="Worldwide",BBC News HD @@ -51,3 +39,7 @@ http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8 https://iptv-all.lanesh4d0w.repl.co/andorra/zoo #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Worldwide",ЛДПР ТВ (1080p) http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8 +#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] +#EXTVLCOPT:http-referrer=http://imn.iq +#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 diff --git a/tests/__data__/expected/.gh-pages/regions/emea.m3u b/tests/__data__/expected/.gh-pages/regions/emea.m3u index c8116aef1..6b40ee03c 100644 --- a/tests/__data__/expected/.gh-pages/regions/emea.m3u +++ b/tests/__data__/expected/.gh-pages/regions/emea.m3u @@ -1,8 +1,4 @@ #EXTM3U -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Undefined",ATV https://iptv-all.lanesh4d0w.repl.co/andorra/atv #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Undefined",Zoo (720p) diff --git a/tests/__data__/expected/.gh-pages/regions/eur.m3u b/tests/__data__/expected/.gh-pages/regions/eur.m3u index c8116aef1..6b40ee03c 100644 --- a/tests/__data__/expected/.gh-pages/regions/eur.m3u +++ b/tests/__data__/expected/.gh-pages/regions/eur.m3u @@ -1,8 +1,4 @@ #EXTM3U -#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Undefined",ATV https://iptv-all.lanesh4d0w.repl.co/andorra/atv #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Undefined",Zoo (720p) diff --git a/tests/__data__/expected/database/db_create.streams.db b/tests/__data__/expected/database/db_create.streams.db deleted file mode 100644 index be143f27e..000000000 --- a/tests/__data__/expected/database/db_create.streams.db +++ /dev/null @@ -1,7 +0,0 @@ -{"line":2,"channel":"ATV.ad","quality":"720p","label":"Offline","name":"ATV","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"k4XpZHQAyqyTbcf0"} -{"line":2,"channel":"LibyasChannel.ly","quality":"","label":"","name":"Libyas Channel","filepath":"ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","httpReferrer":"","userAgent":"","_id":"ki4YjAoNNoIY8sSm"} -{"line":2,"channel":"","quality":"720p","label":"","name":"1A Network","filepath":"unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","httpReferrer":"","userAgent":"","_id":"IZpCJjjWPaBYh7Dr"} -{"line":2,"channel":"","quality":"720p","label":"","name":"Fox Sports 2 Asia (Thai)","filepath":"us_blocked.m3u","url":"https://example.com/playlist.m3u8","httpReferrer":"","userAgent":"","_id":"6c4J4vs8K69wMJ7S"} -{"line":4,"channel":"","quality":"","label":"","name":"TVN","filepath":"us_blocked.m3u","url":"https://example.com/playlist2.m3u8","httpReferrer":"","userAgent":"","_id":"xEmbX384v3t3F5Wg"} -{"line":6,"channel":"EverydayHeroes.us","quality":"720p","label":"","name":"Everyday Heroes","filepath":"us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","httpReferrer":"","userAgent":"","_id":"BZsRPt8VS4kIJnfi"} -{"line":2,"channel":"qib22lAq1L.us","quality":"720p","label":"","name":"ABC","filepath":"wrong_id.m3u","url":"https://example.com/playlist2.m3u8","httpReferrer":"","userAgent":"","_id":"eFUlUnST5zJSBWAF"} diff --git a/tests/__data__/expected/logs/generators.log b/tests/__data__/expected/logs/generators.log index eb311d382..4663f9103 100644 --- a/tests/__data__/expected/logs/generators.log +++ b/tests/__data__/expected/logs/generators.log @@ -32,14 +32,13 @@ {"filepath":"subdivisions/ca-on.m3u","count":1} {"filepath":"countries/in.m3u","count":1} {"filepath":"countries/ru.m3u","count":1} -{"filepath":"countries/uk.m3u","count":1} {"filepath":"countries/int.m3u","count":1} {"filepath":"index.category.m3u","count":8} {"filepath":"index.country.m3u","count":7} {"filepath":"index.language.m3u","count":7} {"filepath":"index.m3u","count":7} {"filepath":"index.nsfw.m3u","count":8} -{"filepath":"index.region.m3u","count":23} +{"filepath":"index.region.m3u","count":21} {"filepath":"languages/eng.m3u","count":1} {"filepath":"languages/rus.m3u","count":1} {"filepath":"languages/cat.m3u","count":1} @@ -54,8 +53,8 @@ {"filepath":"regions/cas.m3u","count":0} {"filepath":"regions/cenamer.m3u","count":0} {"filepath":"regions/cis.m3u","count":1} -{"filepath":"regions/emea.m3u","count":4} -{"filepath":"regions/eur.m3u","count":4} +{"filepath":"regions/emea.m3u","count":3} +{"filepath":"regions/eur.m3u","count":3} {"filepath":"regions/hispam.m3u","count":0} {"filepath":"regions/lac.m3u","count":0} {"filepath":"regions/latam.m3u","count":0} diff --git a/tests/__data__/expected/streams/ad.m3u b/tests/__data__/expected/streams/ad.m3u deleted file mode 100644 index 5007b9ec0..000000000 --- a/tests/__data__/expected/streams/ad.m3u +++ /dev/null @@ -1,5 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="AndorraTV.ad" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",ATV (720p) [Offline] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -https://iptv-all.lanesh4d0w.repl.co/andorra/atv diff --git a/tests/__data__/expected/streams/af.m3u b/tests/__data__/expected/streams/af.m3u deleted file mode 100644 index 976a1d532..000000000 --- a/tests/__data__/expected/streams/af.m3u +++ /dev/null @@ -1,5 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="KayhanTV.af",Kayhan TV -http://208.93.117.113/live/Stream1/playlist.m3u8 -#EXTINF:-1 tvg-id="Sharq.af",Sharq -http://51.210.199.50/hls/stream.m3u8 diff --git a/tests/__data__/expected/streams/aw.m3u b/tests/__data__/expected/streams/aw.m3u deleted file mode 100644 index 3c2c2eb89..000000000 --- a/tests/__data__/expected/streams/aw.m3u +++ /dev/null @@ -1,6 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="Telearuba.aw",Telearuba (720p) -http://cdn.setar.aw:1935/Telearuba/smil:telearuba.smil/playlist.m3u8 -#EXTINF:-1 tvg-id="Telearuba.aw" user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",Telearuba (480p) [Not 24/7] -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36 -https://backend-server-dot-telearuba-app.appspot.com/media/livestream13/playlist.m3u8 diff --git a/tests/__data__/expected/streams/es.m3u b/tests/__data__/expected/streams/es.m3u deleted file mode 100644 index 9b94dc893..000000000 --- a/tests/__data__/expected/streams/es.m3u +++ /dev/null @@ -1,7 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="",Caillou -https://dhx-caillou-1-es.samsung.wurl.tv/playlist.m3u8 -#EXTINF:-1 tvg-id="",iHola Play -https://rakuten-hola-2-es.samsung.wurl.tv/playlist.m3u8 -#EXTINF:-1 tvg-id="",Planeta Junior TV -https://deaplaneta-planetakidz-1-es.samsung.wurl.tv/playlist.m3u8 diff --git a/tests/__data__/expected/streams/ru.m3u b/tests/__data__/expected/streams/ru.m3u deleted file mode 100644 index 17ed7be44..000000000 --- a/tests/__data__/expected/streams/ru.m3u +++ /dev/null @@ -1,9 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) -http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 -#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) -http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 -#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) -http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 -#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) -https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?advertisingId=&appName=web&deviceId=5ca525b650be2571e3943c63 diff --git a/tests/__data__/expected/streams_format/in.m3u b/tests/__data__/expected/streams_format/in.m3u new file mode 100644 index 000000000..dace15a30 --- /dev/null +++ b/tests/__data__/expected/streams_format/in.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="",Manorama News +https://ythls.onrender.com/channel/UCP0uG-mcMImgKnJz-VjJZmQ.m3u8 diff --git a/tests/__data__/expected/streams/nl.m3u b/tests/__data__/expected/streams_format/nl.m3u similarity index 63% rename from tests/__data__/expected/streams/nl.m3u rename to tests/__data__/expected/streams_format/nl.m3u index 0be4e08e1..ff3188ba0 100644 --- a/tests/__data__/expected/streams/nl.m3u +++ b/tests/__data__/expected/streams_format/nl.m3u @@ -1,8 +1,4 @@ #EXTM3U -#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (1080p) [Geo-blocked] -http://stream.tvtap.net:8081/live/nl-npo1.stream/30fps.m3u8 -#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (1080p) [Geo-blocked] -http://stream.tvtap.net:8081/live/nl-npo1.stream/60fps.m3u8 #EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (342p) [Geo-blocked] http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8 #EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (342p) diff --git a/tests/__data__/expected/streams/in.m3u b/tests/__data__/expected/streams_update/in.m3u similarity index 100% rename from tests/__data__/expected/streams/in.m3u rename to tests/__data__/expected/streams_update/in.m3u diff --git a/tests/__data__/expected/streams/my.m3u b/tests/__data__/expected/streams_update/my.m3u similarity index 100% rename from tests/__data__/expected/streams/my.m3u rename to tests/__data__/expected/streams_update/my.m3u diff --git a/tests/__data__/expected/streams/uk.m3u b/tests/__data__/expected/streams_update/uk.m3u similarity index 100% rename from tests/__data__/expected/streams/uk.m3u rename to tests/__data__/expected/streams_update/uk.m3u diff --git a/tests/__data__/expected/streams/us.m3u b/tests/__data__/expected/streams_update/us.m3u similarity index 100% rename from tests/__data__/expected/streams/us.m3u rename to tests/__data__/expected/streams_update/us.m3u diff --git a/tests/__data__/input/data/streams.json b/tests/__data__/input/data/streams.json deleted file mode 100644 index 5b97f3e1e..000000000 --- a/tests/__data__/input/data/streams.json +++ /dev/null @@ -1 +0,0 @@ -[{"channel":"TUTV.us","url":"https://livestream.telvue.com/templeuni1/f7b44cfafd5c52223d5498196c8a2e7b.sdp/playlist.m3u8","http_referrer":null,"user_agent":null}] \ No newline at end of file diff --git a/tests/__data__/input/database/api_generate.streams.db b/tests/__data__/input/database/api_generate.streams.db deleted file mode 100644 index 110767d61..000000000 --- a/tests/__data__/input/database/api_generate.streams.db +++ /dev/null @@ -1,7 +0,0 @@ -{"line": 2,"quality":null,"label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"} -{"line": 2,"quality":null,"label":null,"name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"} -{"line": 2,"quality":null,"label":null,"name":"ATV","channel":"AndorraTV.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error"} -{"line": 2,"quality":null,"label":null,"name":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} -{"line": 2,"quality":null,"label":null,"name":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"} -{"line": 2,"quality":null,"label":null,"name":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"} -{"line": 2,"quality":null,"label":null,"name":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} diff --git a/tests/__data__/input/database/playlist_generate.streams.db b/tests/__data__/input/database/playlist_generate.streams.db deleted file mode 100644 index 4367a38c1..000000000 --- a/tests/__data__/input/database/playlist_generate.streams.db +++ /dev/null @@ -1,14 +0,0 @@ -{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF0"} -{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/timeout.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF1"} -{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF2"} -{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/error.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF3"} -{"line": 2,"quality":"720p","label":"Geo-blocked","name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"3TbieV1ptnZVCId5"} -{"line": 2,"quality":null,"label":null,"name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","httpReferrer":null,"userAgent":null,"_id":"3TbieV1ptnZVCIdn"} -{"line": 2,"quality":null,"label":null,"name":"ATV","channel":"AndorraTV.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":null,"userAgent":null,"_id":"I6cjG2xCBRFFP44z"} -{"line": 2,"quality":"720p","label":"Not 24/7","name":"Andorra TV","channel":"","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"WTbieV1ptnZVCIdn"} -{"line": 2,"quality":null,"label":null,"name":"Visit-X TV","channel":"VisitXTV.nl","filepath":"nl.m3u","url":"https://stream.visit-x.tv/vxtv/ngrp:live_all/30fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF5"} -{"line": 2,"quality":null,"label":null,"name":"Visit-X TV","channel":"VisitXTV.nl","filepath":"nl.m3u","url":"https://stream.visit-x.tv/vxtv/ngrp:live_all/60fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF6"} -{"line": 2,"quality":null,"label":null,"name":"Daawah TV","channel":"","filepath":"in.m3u","url":"http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF9"} -{"line": 2,"quality":null,"label":null,"name":"Meteomedia","channel":"MeteoMedia.ca","filepath":"in.m3u","url":"http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgP49"} -{"line": 2,"quality":"480p","label":null,"name":"Zoo","channel":"Zoo.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/zoo?480","httpReferrer":null,"userAgent":null,"_id":"I6cjG2xCBRFFP4s3"} -{"line": 2,"quality":"720p","label":null,"name":"Zoo","channel":"Zoo.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/zoo","httpReferrer":null,"userAgent":null,"_id":"I6cjG2xCBRFFP4sz"} diff --git a/tests/__data__/input/database/playlist_update.streams.db b/tests/__data__/input/database/playlist_update.streams.db deleted file mode 100644 index 898b99357..000000000 --- a/tests/__data__/input/database/playlist_update.streams.db +++ /dev/null @@ -1,21 +0,0 @@ -{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?deviceId=5ca525b650be2571e3943c63&appName=web&advertisingId=","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF3"} -{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF0"} -{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF1"} -{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF2"} -{"line": 2,"name":"BBC News HD","quality":"720p","label":"Not 24/7","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","httpReferrer":null,"userAgent":null,"_id":"3TbieV1ptnZVCIdn"} -{"line": 2,"quality":"720p","label":"Offline","name":"ATV","channel":"AndorraTV.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"I6cjG2xCBRFFP4sz"} -{"line": 2,"quality":"480p","label":"Geo-blocked","name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"WTbieV1ptnZVCIdn"} -{"line": 2,"quality":null,"label":null,"name":"Kayhan TV","channel":"KayhanTV.af","filepath":"af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"cFFpFVzSn6xFMUF3"} -{"line": 2,"quality":null,"label":null,"name":"Sharq","channel":"Sharq.af","filepath":"af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","httpReferrer":null,"userAgent":null,"_id":"u7iyA6cjtf1iWWAZ"} -{"line": 2,"quality":"342p","label":"Geo-blocked","channel":"NPO1.nl","name":"NPO 1","filepath":"nl.m3u","url":"http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8","httpReferrer":null,"userAgent":null,"_id":"mvUyDVuS5gc8gLJV"} -{"line": 2,"quality":"1080p","label":"Geo-blocked","channel":"NPO1.nl","name":"NPO 1","filepath":"nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo1.stream/30fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"8WVbsxsYeOL7kHQl"} -{"line": 2,"quality":"1080p","label":"Geo-blocked","channel":"NPO1.nl","name":"NPO 1","filepath":"nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo1.stream/60fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"8WVbsxsYeOL7kHQB"} -{"line": 2,"quality":"342p","label":null,"channel":"NPO2.nl","name":"NPO 2","filepath":"nl.m3u","url":"http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8","httpReferrer":null,"userAgent":null,"_id":"2p1TNGO0mF0MJOGy"} -{"line": 2,"quality":"302p","label":"Geo-blocked","channel":"NPO2.nl","name":"NPO 2","filepath":"nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo2.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"nhL85BL7YM5OR7cn"} -{"line": 2,"quality":null,"label":null,"name":"Tele 2000","channel":"Tele2000.pe","filepath":"pe.m3u","url":"https://servilive.com:3126/live/tele2000live.m3u8","httpReferrer":"https://example2.com/","userAgent":null,"_id":"cF0pFVzSn6xFMUF3"} -{"line": 2,"quality":null,"label":null,"name":"Planeta Junior TV","channel":"","filepath":"es.m3u","url":"https://deaplaneta-planetakidz-1-es.samsung.wurl.tv/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"1BT8btby3mmsgPF0"} -{"line": 2,"quality":null,"label":null,"name":"Caillou","channel":"","filepath":"es.m3u","url":"https://dhx-caillou-1-es.samsung.wurl.tv/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"3BT8btby3mmsgPF0"} -{"line": 2,"quality":null,"label":null,"name":"iHola Play","channel":"","filepath":"es.m3u","url":"https://rakuten-hola-2-es.samsung.wurl.tv/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2BT8btby3mmsgPF0"} -{"line": 2,"quality":"720p","label":null,"name":"Telearuba","channel":"Telearuba.aw","filepath":"aw.m3u","url":"http://cdn.setar.aw:1935/Telearuba/smil:telearuba.smil/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"6BT8btby3mmsgPF0"} -{"line": 2,"quality":"480p","label":"Not 24/7","name":"Telearuba","channel":"Telearuba.aw","filepath":"aw.m3u","url":"https://backend-server-dot-telearuba-app.appspot.com/media/livestream13/playlist.m3u8","httpReferrer":null,"userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36","_id":"4BT8btby3mmsgPF0"} -{"line": 2,"quality":"","label":"","name":"Telearuba","channel":"","filepath":"bg.m3u","url":"https://ythls.onrender.com/channel/UC40TUSUx490U5uR1lZt3Ajgm3u8","httpReferrer":null,"userAgent":null,"_id":"4BT8btby3mmsgSF0"} diff --git a/tests/__data__/input/streams/ad.m3u b/tests/__data__/input/streams/ad.m3u deleted file mode 100644 index 8bde22790..000000000 --- a/tests/__data__/input/streams/ad.m3u +++ /dev/null @@ -1,5 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="ATV.ad" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",ATV (720p) [Offline] -#EXTVLCOPT:http-referrer=http://imn.iq -#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 -https://iptv-all.lanesh4d0w.repl.co/andorra/atv \ No newline at end of file diff --git a/tests/__data__/input/streams/ly.m3u b/tests/__data__/input/streams/ly.m3u deleted file mode 100644 index 8dd73939a..000000000 --- a/tests/__data__/input/streams/ly.m3u +++ /dev/null @@ -1,3 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="LibyasChannel.ly",Libyas Channel -https://master.starmena-cloud.com/hls/libyas.m3u8 diff --git a/tests/__data__/input/streams/unsorted.m3u b/tests/__data__/input/streams/unsorted.m3u deleted file mode 100644 index 0cb74b761..000000000 --- a/tests/__data__/input/streams/unsorted.m3u +++ /dev/null @@ -1,3 +0,0 @@ -#EXTM3U -#EXTINF:-1 tvg-id="",1A Network (720p) -https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8 diff --git a/tests/__data__/input/streams_format/in.m3u b/tests/__data__/input/streams_format/in.m3u new file mode 100644 index 000000000..da450b06f --- /dev/null +++ b/tests/__data__/input/streams_format/in.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="mn.in",Manorama News +https://ythls.onrender.com/channel/UCP0uG-mcMImgKnJz-VjJZmQ.m3u8 diff --git a/tests/__data__/input/streams_format/nl.m3u b/tests/__data__/input/streams_format/nl.m3u new file mode 100644 index 000000000..3ba85b77e --- /dev/null +++ b/tests/__data__/input/streams_format/nl.m3u @@ -0,0 +1,9 @@ +#EXTM3U +#EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (302p) [Geo-blocked] +http://stream.tvtap.net:8081/live/nl-npo2.stream/playlist.m3u8? +#EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (342p) +http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8 +#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (342p) [Geo-blocked] +http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8 +#EXTINF:-1 tvg-id="",NPO 2 (Duplicate) +http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8 diff --git a/tests/__data__/input/streams_generate/ad.m3u b/tests/__data__/input/streams_generate/ad.m3u new file mode 100644 index 000000000..aed019154 --- /dev/null +++ b/tests/__data__/input/streams_generate/ad.m3u @@ -0,0 +1,5 @@ +#EXTM3U +#EXTINF:-1 tvg-id="Zoo.ad",Zoo (720p) +https://iptv-all.lanesh4d0w.repl.co/andorra/zoo +#EXTINF:-1 tvg-id="AndorraTV.ad",ATV +https://iptv-all.lanesh4d0w.repl.co/andorra/atv diff --git a/tests/__data__/input/streams_generate/ca.m3u b/tests/__data__/input/streams_generate/ca.m3u new file mode 100644 index 000000000..9c198b682 --- /dev/null +++ b/tests/__data__/input/streams_generate/ca.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="MeteoMedia.ca",Meteomedia +http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8 diff --git a/tests/__data__/input/streams_generate/in.m3u b/tests/__data__/input/streams_generate/in.m3u new file mode 100644 index 000000000..62594eb4b --- /dev/null +++ b/tests/__data__/input/streams_generate/in.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="",Daawah TV +http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8 diff --git a/tests/__data__/input/streams_generate/uk.m3u b/tests/__data__/input/streams_generate/uk.m3u new file mode 100644 index 000000000..5e8c7318e --- /dev/null +++ b/tests/__data__/input/streams_generate/uk.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8 diff --git a/tests/__data__/input/streams_generate/unsorted.m3u b/tests/__data__/input/streams_generate/unsorted.m3u new file mode 100644 index 000000000..1a9c40b9f --- /dev/null +++ b/tests/__data__/input/streams_generate/unsorted.m3u @@ -0,0 +1,9 @@ +#EXTM3U +#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) +http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8 +#EXTINF:-1 tvg-id="VisitXTV.nl",Visit-X TV +https://stream.visit-x.tv/vxtv/ngrp:live_all/30fps.m3u8 +#EXTINF:-1 tvg-id="" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7] +#EXTVLCOPT:http-referrer=http://imn.iq +#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8 diff --git a/tests/__data__/input/streams_report/us.m3u b/tests/__data__/input/streams_report/us.m3u new file mode 100644 index 000000000..92cee65b0 --- /dev/null +++ b/tests/__data__/input/streams_report/us.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="",TUTV +https://livestream.telvue.com/templeuni1/f7b44cfafd5c52223d5498196c8a2e7b.sdp/playlist.m3u8 diff --git a/tests/__data__/input/streams_update/br.m3u b/tests/__data__/input/streams_update/br.m3u new file mode 100644 index 000000000..5d1b51ed4 --- /dev/null +++ b/tests/__data__/input/streams_update/br.m3u @@ -0,0 +1,6 @@ +#EXTM3U +#EXTINF:-1 tvg-id="",VTV +https://ythls.onrender.com/channel/UC40TUSUx490U5uR1lZt3Ajgm3u8 +#EXTINF:-1 tvg-id="",Tele2000 +#EXTVLCOPT:http-referrer=https://example2.com/ +https://servilive.com:3126/live/tele2000live.m3u8 diff --git a/tests/__data__/input/streams_update/uk.m3u b/tests/__data__/input/streams_update/uk.m3u new file mode 100644 index 000000000..c1deeea97 --- /dev/null +++ b/tests/__data__/input/streams_update/uk.m3u @@ -0,0 +1,5 @@ +#EXTM3U +#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD (720p) [Not 24/7] +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8 +#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD (480p) [Geo-blocked] +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8 diff --git a/tests/__data__/input/streams/us_blocked.m3u b/tests/__data__/input/streams_validate/us_blocked.m3u similarity index 100% rename from tests/__data__/input/streams/us_blocked.m3u rename to tests/__data__/input/streams_validate/us_blocked.m3u diff --git a/tests/__data__/input/streams/wrong_id.m3u b/tests/__data__/input/streams_validate/wrong_id.m3u similarity index 100% rename from tests/__data__/input/streams/wrong_id.m3u rename to tests/__data__/input/streams_validate/wrong_id.m3u diff --git a/tests/commands/api/generate.test.ts b/tests/commands/api/generate.test.ts index 74a70e11d..fe3c1215d 100644 --- a/tests/commands/api/generate.test.ts +++ b/tests/commands/api/generate.test.ts @@ -3,14 +3,9 @@ import fs from 'fs-extra' beforeEach(() => { fs.emptyDirSync('tests/__data__/output') - fs.mkdirSync('tests/__data__/output/database') - fs.copyFileSync( - 'tests/__data__/input/database/api_generate.streams.db', - 'tests/__data__/output/database/streams.db' - ) const stdout = execSync( - 'DB_DIR=tests/__data__/output/database API_DIR=tests/__data__/output/.api npm run api:generate', + 'STREAMS_DIR=tests/__data__/input/streams_generate API_DIR=tests/__data__/output/.api npm run api:generate', { encoding: 'utf8' } ) }) diff --git a/tests/commands/database/create.test.ts b/tests/commands/database/create.test.ts deleted file mode 100644 index 4ca4d4023..000000000 --- a/tests/commands/database/create.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import * as fs from 'fs-extra' -import * as path from 'path' -import { execSync } from 'child_process' -import * as _ from 'lodash' - -beforeEach(() => { - fs.emptyDirSync('tests/__data__/output') - fs.mkdirSync('tests/__data__/output/database') - - const stdout = execSync( - 'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams npm run db:create', - { encoding: 'utf8' } - ) -}) - -it('can create database', () => { - let output = content('tests/__data__/output/database/streams.db') - let expected = content('tests/__data__/expected/database/db_create.streams.db') - - output = output.map(i => { - i._id = null - return i - }) - expected = expected.map(i => { - i._id = null - return i - }) - - expect(_.orderBy(output, 'name')).toMatchObject( - expect.arrayContaining(_.orderBy(expected, 'name')) - ) -}) - -function content(filepath: string) { - const data = fs.readFileSync(path.resolve(filepath), { - encoding: 'utf8' - }) - - return data - .split('\n') - .filter(l => l) - .map(l => { - return JSON.parse(l) - }) -} diff --git a/tests/commands/playlist/format.test.ts b/tests/commands/playlist/format.test.ts new file mode 100644 index 000000000..0c867dd36 --- /dev/null +++ b/tests/commands/playlist/format.test.ts @@ -0,0 +1,30 @@ +import { execSync } from 'child_process' +import * as fs from 'fs-extra' +import { glob } from 'glob' + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') + fs.copySync('tests/__data__/input/streams_format', 'tests/__data__/output/streams') +}) + +it('can format playlists', () => { + const stdout = execSync('STREAMS_DIR=tests/__data__/output/streams npm run playlist:format', { + encoding: 'utf8' + }) + + const files = glob + .sync('tests/__data__/expected/streams_format/*.m3u') + .map(f => f.replace('tests/__data__/expected/streams_format/', '')) + + files.forEach(filepath => { + expect(content(`output/streams/${filepath}`), filepath).toBe( + content(`expected/streams_format/${filepath}`) + ) + }) +}) + +function content(filepath: string) { + return fs.readFileSync(`tests/__data__/${filepath}`, { + encoding: 'utf8' + }) +} diff --git a/tests/commands/playlist/generate.test.ts b/tests/commands/playlist/generate.test.ts index 1e2d38abe..8c6d80d6a 100644 --- a/tests/commands/playlist/generate.test.ts +++ b/tests/commands/playlist/generate.test.ts @@ -4,13 +4,9 @@ import * as glob from 'glob' beforeEach(() => { fs.emptyDirSync('tests/__data__/output') - fs.copyFileSync( - 'tests/__data__/input/database/playlist_generate.streams.db', - 'tests/__data__/output/streams.db' - ) const stdout = execSync( - 'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.gh-pages LOGS_DIR=tests/__data__/output/logs npm run playlist:generate', + 'STREAMS_DIR=tests/__data__/input/streams_generate DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.gh-pages LOGS_DIR=tests/__data__/output/logs npm run playlist:generate', { encoding: 'utf8' } ) }) diff --git a/tests/commands/playlist/update.test.ts b/tests/commands/playlist/update.test.ts index 01cad1b84..4670b9452 100644 --- a/tests/commands/playlist/update.test.ts +++ b/tests/commands/playlist/update.test.ts @@ -4,15 +4,12 @@ import { glob } from 'glob' beforeEach(() => { fs.emptyDirSync('tests/__data__/output') - fs.copyFileSync( - 'tests/__data__/input/database/playlist_update.streams.db', - 'tests/__data__/output/streams.db' - ) + fs.copySync('tests/__data__/input/streams_update', 'tests/__data__/output/streams') }) it('can format playlists', () => { const stdout = execSync( - 'DEBUG=true DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/output/streams DB_DIR=tests/__data__/output npm run playlist:update --silent', + 'DEBUG=true DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/output/streams npm run playlist:update --silent', { encoding: 'utf8' } @@ -21,11 +18,13 @@ it('can format playlists', () => { expect(stdout).toBe(`OUTPUT=closes #14151, closes #14110, closes #14179, closes #14178\n`) const files = glob - .sync('tests/__data__/expected/streams/*.m3u') - .map(f => f.replace('tests/__data__/expected/', '')) + .sync('tests/__data__/expected/streams_update/*.m3u') + .map(f => f.replace('tests/__data__/expected/streams_update/', '')) files.forEach(filepath => { - expect(content(`output/${filepath}`), filepath).toBe(content(`expected/${filepath}`)) + expect(content(`output/streams/${filepath}`), filepath).toBe( + content(`expected/streams_update/${filepath}`) + ) }) }) diff --git a/tests/commands/playlist/validate.test.ts b/tests/commands/playlist/validate.test.ts index 7f702bb36..d0c6d1396 100644 --- a/tests/commands/playlist/validate.test.ts +++ b/tests/commands/playlist/validate.test.ts @@ -3,7 +3,7 @@ import { execSync } from 'child_process' it('show an error if channel name in the blocklist', () => { try { const stdout = execSync( - 'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams npm run playlist:validate -- tests/__data__/input/streams/us_blocked.m3u', + 'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams_validate npm run playlist:validate -- us_blocked.m3u', { encoding: 'utf8' } @@ -14,7 +14,7 @@ it('show an error if channel name in the blocklist', () => { expect(error.status).toBe(1) expect( error.stdout.includes( - `loading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2 error "Fox Sports 2 Asia (Thai)" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n` + `us_blocked.m3u\n 2 error "Fox Sports 2 Asia (Thai)" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n` ) ).toBe(true) } @@ -22,7 +22,7 @@ it('show an error if channel name in the blocklist', () => { it('show a warning if channel has wrong id', () => { const stdout = execSync( - 'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams npm run playlist:validate -- tests/__data__/input/streams/wrong_id.m3u', + 'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams_validate npm run playlist:validate -- wrong_id.m3u', { encoding: 'utf8' } @@ -30,7 +30,7 @@ it('show a warning if channel has wrong id', () => { expect( stdout.includes( - `loading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2 warning "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n` + `wrong_id.m3u\n 2 warning "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n` ) ).toBe(true) }) diff --git a/tests/commands/report/create.test.ts b/tests/commands/report/create.test.ts index b6002eeba..55199f81c 100644 --- a/tests/commands/report/create.test.ts +++ b/tests/commands/report/create.test.ts @@ -1,9 +1,12 @@ import { execSync } from 'child_process' it('can create report', () => { - const stdout = execSync('DATA_DIR=tests/__data__/input/data npm run report:create', { - encoding: 'utf8' - }) + const stdout = execSync( + 'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams_report npm run report:create', + { + encoding: 'utf8' + } + ) expect( stdout.includes(` diff --git a/yarn.lock b/yarn.lock index 0df54a9ed..79504afd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -670,20 +670,6 @@ resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@seald-io/binary-search-tree@^1.0.3": - version "1.0.3" - resolved "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.3.tgz" - integrity sha512-qv3jnwoakeax2razYaMsGI/luWdliBLHTdC6jU55hQt1hcFqzauH/HsBollQ7IR4ySTtYhT+xyHoijpA16C+tA== - -"@seald-io/nedb@^4.0.2": - version "4.0.2" - resolved "https://registry.npmjs.org/@seald-io/nedb/-/nedb-4.0.2.tgz" - integrity sha512-gJ91fT1sgh2cLXYVcTSh7khZ8LdemI8+SojCdpZ5wy+DUQ4fSrEwGqOwbdV49NDs2BBO6GeBpSb8CnhG2IW1rw== - dependencies: - "@seald-io/binary-search-tree" "^1.0.3" - localforage "^1.9.0" - util "^0.12.4" - "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz" @@ -947,11 +933,6 @@ async@^3.2.4: resolved "https://registry.npmjs.org/async/-/async-3.2.4.tgz" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== -available-typed-arrays@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz" - integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== - babel-jest@^29.0.0, babel-jest@^29.6.4: version "29.6.4" resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz" @@ -1074,14 +1055,6 @@ buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -call-bind@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz" - integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== - dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.2" - callsites@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz" @@ -1431,13 +1404,6 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -for-each@^0.3.3: - version "0.3.3" - resolved "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz" - integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw== - dependencies: - is-callable "^1.1.3" - foreground-child@^3.1.0: version "3.1.1" resolved "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz" @@ -1489,16 +1455,6 @@ get-caller-file@^2.0.5: resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.2, get-intrinsic@^1.1.3: - version "1.2.1" - resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz" - integrity sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw== - dependencies: - function-bind "^1.1.1" - has "^1.0.3" - has-proto "^1.0.1" - has-symbols "^1.0.3" - get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz" @@ -1597,13 +1553,6 @@ globby@^6.1.0: pify "^2.0.0" pinkie-promise "^2.0.0" -gopd@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz" - integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== - dependencies: - get-intrinsic "^1.1.3" - graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.9: version "4.2.9" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz" @@ -1619,23 +1568,6 @@ has-flag@^4.0.0: resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has-proto@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz" - integrity sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg== - -has-symbols@^1.0.2, has-symbols@^1.0.3: - version "1.0.3" - resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz" - integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== - -has-tostringtag@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz" - integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== - dependencies: - has-symbols "^1.0.2" - has@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/has/-/has-1.0.3.tgz" @@ -1653,11 +1585,6 @@ human-signals@^2.1.0: resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== -immediate@~3.0.5: - version "3.0.6" - resolved "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz" - integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps= - import-local@^3.0.2: version "3.1.0" resolved "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz" @@ -1679,7 +1606,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@^2.0.3, inherits@2: +inherits@2: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -1692,24 +1619,11 @@ iptv-playlist-parser@^0.12.3: is-valid-path "^0.1.1" validator "^13.7.0" -is-arguments@^1.0.4: - version "1.1.1" - resolved "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz" - integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA== - dependencies: - call-bind "^1.0.2" - has-tostringtag "^1.0.0" - is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= -is-callable@^1.1.3: - version "1.2.7" - resolved "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz" - integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== - is-core-module@^2.13.0: version "2.13.0" resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz" @@ -1732,13 +1646,6 @@ is-generator-fn@^2.0.0: resolved "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz" integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ== -is-generator-function@^1.0.7: - version "1.0.10" - resolved "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz" - integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A== - dependencies: - has-tostringtag "^1.0.0" - is-glob@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz" @@ -1768,13 +1675,6 @@ is-stream@^2.0.0: resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== -is-typed-array@^1.1.3: - version "1.1.12" - resolved "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz" - integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg== - dependencies: - which-typed-array "^1.1.11" - is-valid-path@^0.1.1: version "0.1.1" resolved "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz" @@ -2272,13 +2172,6 @@ leven@^3.1.0: resolved "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz" integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A== -lie@3.1.1: - version "3.1.1" - resolved "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz" - integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4= - dependencies: - immediate "~3.0.5" - lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz" @@ -2294,13 +2187,6 @@ load-json-file@^4.0.0: pify "^3.0.0" strip-bom "^3.0.0" -localforage@^1.9.0: - version "1.10.0" - resolved "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz" - integrity sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg== - dependencies: - lie "3.1.1" - locate-path@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz" @@ -3008,17 +2894,6 @@ universalify@^2.0.0: resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz" integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== -util@^0.12.4: - version "0.12.5" - resolved "https://registry.npmjs.org/util/-/util-0.12.5.tgz" - integrity sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA== - dependencies: - inherits "^2.0.3" - is-arguments "^1.0.4" - is-generator-function "^1.0.7" - is-typed-array "^1.1.3" - which-typed-array "^1.1.2" - v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz" @@ -3058,17 +2933,6 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -which-typed-array@^1.1.11, which-typed-array@^1.1.2: - version "1.1.11" - resolved "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz" - integrity sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew== - dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.2" - for-each "^0.3.3" - gopd "^1.0.1" - has-tostringtag "^1.0.0" - which@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"