Merge pull request #14243 from iptv-org/patch-2023.09.4

2 years ago · cd92c9320c
parent f13b9a75ff 0739182d6d
commit cd92c9320c
68 changed files with 452 additions and 950 deletions
--- a/.github/workflows/format.yml
+++ b/.github/workflows/format.yml
@ -0,0 +1,62 @@
+name: format
+on:
+  workflow_dispatch:
+  # pull_request:
+  #   types:
+  #     - closed
+jobs:
+  main:
+    # if: ${{ github.event.pull_request.merged == true }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: getsentry/action-github-app-token@v2
+        if: ${{ !env.ACT }}
+        id: create-app-token
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+      - uses: actions/checkout@v3
+        if: ${{ !env.ACT }}
+        with:
+          fetch-depth: 2
+          token: ${{ steps.create-app-token.outputs.token }}
+      - name: setup git
+        run: |
+          git config user.name "iptv-bot[bot]"
+          git config user.email "84861620+iptv-bot[bot]@users.noreply.github.com"
+      - uses: tj-actions/changed-files@v35
+        id: files
+        with:
+          files: streams/*.m3u
+      - name: download data from api
+        run: |
+          mkdir -p temp/data
+          curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json
+          curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json
+      - uses: actions/setup-node@v3
+        if: ${{ !env.ACT }}
+        with:
+          node-version: 18
+          cache: 'npm'
+      - name: install dependencies
+        run: npm install
+      - name: format internal playlists
+        run: npm run playlist:format
+      - name: check internal playlists
+        run: |
+          npm run playlist:lint
+          npm run playlist:validate
+      - run: git status
+      - name: commit changes to /streams
+        run: |
+          git add streams
+          git status
+          git commit -m "[Bot] Format /streams" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [format](https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}) workflow." --no-verify
+      - name: push all changes to the repository
+        if: ${{ !env.ACT && github.ref == 'refs/heads/master' }}
+        run: git push
--- a/.github/workflows/update.yml
+++ b/.github/workflows/update.yml
@ -33,8 +33,6 @@ jobs:
        run: npm install
      - name: load api data
        run: npm run api:load
-      - name: setup database
-        run: npm run db:create
      - name: update internal playlists
        run: npm run playlist:update --silent >> $GITHUB_OUTPUT
        id: playlist-update
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -123,21 +123,22 @@ For scripts to work, you must have [Node.js](https://nodejs.org/en) installed on
 To run scripts use the `npm run <script-name>` command.

 - `act:check`: allows to run the [check](https://github.com/iptv-org/iptv/blob/master/.github/workflows/check.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
+- `act:format`: allows to test the [format](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
 - `act:update`: allows to test the [update](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
 - `api:load`: downloads the latest channel and stream data from the [iptv-org/api](https://github.com/iptv-org/api).
 - `api:generate`: generates a JSON file with all streams for the [iptv-org/api](https://github.com/iptv-org/api) repository.
 - `api:deploy`: allows to manually upload a JSON file created via `api:generate` to the [iptv-org/api](https://github.com/iptv-org/api) repository. To run the script you must provide your [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) with write access to the repository.
- `db:create`: сreates a temporary file `temp/database/streams.db` containing all links from the [/streams]() folder.
- `playlist:update`: triggers an update of internal playlists. The process involves processing approved requests from issues, URL normalization, and sorting links by channel name, quality, and label.
+- `playlist:format`: formats internal playlists. The process includes [URL normalization](https://en.wikipedia.org/wiki/URI_normalization), duplicate removal, removing invalid id's and sorting links by channel name, quality, and label.
+- `playlist:update`: triggers an update of internal playlists. The process involves processing approved requests from issues.
+- `playlist:generate`: generates all public playlists.
 - `playlist:validate`: сhecks ids and links in internal playlists for errors.
 - `playlist:lint`: сhecks internal playlists for syntax errors.
- `playlist:generate`: generates all public playlists.
 - `playlist:deploy`: allows to manually publish all generated via `playlist:generate` playlists. To run the script you must provide your [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) with write access to the repository.
 - `readme:update`: updates the list of playlists in [README.md](README.md).
 - `report:create`: shows a list of all current requests and their status.
- `format`: (shorthand) sequentially runs the `db:create` and `db:create` commands.
+- `format`: (shorthand) sequentially runs the `api:load` and `playlist:format` commands.
 - `check`: (shorthand) sequentially runs the `api:load`, `playlist:lint` and `playlist:validate` commands.
- `update`: (shorthand) sequentially runs the `api:load`, `db:create`, `playlist:generate`, `api:generate` and `readme:update` commands.
+- `update`: (shorthand) sequentially runs the `api:load`, `playlist:generate`, `api:generate` and `readme:update` commands.
 - `deploy`: (shorthand) sequentially runs the `playlist:deploy` and `api:deploy` commands.
 - `report`: (shorthand) sequentially runs the `api:load` and `report:create` commands.
 - `test`: runs a test of all the scripts described above.
@ -148,5 +149,6 @@ To automate the run of the scripts described above, we use the [GitHub Actions w

 Each workflow includes its own set of scripts that can be run either manually or in response to an event.

- `check`: sequentially runs the `playlist:check` and `playlist:validate` scripts when a new pull request appears, and blocks the merge if it detects an error in it.
- `update`: every day at 0:00 UTC sequentially runs `api:load`, `db:create`, `playlist:update`, `playlist:lint`, `playlist:validate`, `playlist:generate`, `api:generate` and `readme:update` scripts and deploys the output files if successful.
+- `check`: sequentially runs the `api:load`, `playlist:check` and `playlist:validate` scripts when a new pull request appears, and blocks the merge if it detects an error in it.
+- `format`: sequentially runs `api:load`, `playlist:format`, `playlist:lint` and `playlist:validate` scripts.
+- `update`: every day at 0:00 UTC sequentially runs `api:load`, `playlist:update`, `playlist:lint`, `playlist:validate`, `playlist:generate`, `api:generate` and `readme:update` scripts and deploys the output files if successful.
--- a/act.json
+++ b/act.json
@ -0,0 +1,5 @@
+{
+	"pull_request": {
+		"merged": true
+	}
+}
--- a/package-lock.json
+++ b/package-lock.json
@ -11,7 +11,6 @@
        "@octokit/plugin-paginate-rest": "^7.1.2",
        "@octokit/plugin-rest-endpoint-methods": "^7.1.3",
        "@octokit/types": "^11.1.0",
-        "@seald-io/nedb": "^4.0.2",
        "@types/fs-extra": "^11.0.1",
        "@types/glob": "^8.1.0",
        "@types/jest": "^29.5.4",
@ -1338,21 +1337,6 @@
        "node": ">=14"
      }
    },
-    "node_modules/@seald-io/binary-search-tree": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.3.tgz",
-      "integrity": "sha512-qv3jnwoakeax2razYaMsGI/luWdliBLHTdC6jU55hQt1hcFqzauH/HsBollQ7IR4ySTtYhT+xyHoijpA16C+tA=="
-    },
-    "node_modules/@seald-io/nedb": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/@seald-io/nedb/-/nedb-4.0.2.tgz",
-      "integrity": "sha512-gJ91fT1sgh2cLXYVcTSh7khZ8LdemI8+SojCdpZ5wy+DUQ4fSrEwGqOwbdV49NDs2BBO6GeBpSb8CnhG2IW1rw==",
-      "dependencies": {
-        "@seald-io/binary-search-tree": "^1.0.3",
-        "localforage": "^1.9.0",
-        "util": "^0.12.4"
-      }
-    },
    "node_modules/@sinclair/typebox": {
      "version": "0.27.8",
      "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
@ -1658,17 +1642,6 @@
        "node": ">=0.10.0"
      }
    },
-    "node_modules/available-typed-arrays": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
-      "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/babel-jest": {
      "version": "29.6.4",
      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz",
@ -1855,18 +1828,6 @@
      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
      "peer": true
    },
-    "node_modules/call-bind": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
-      "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
-      "dependencies": {
-        "function-bind": "^1.1.1",
-        "get-intrinsic": "^1.0.2"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/callsites": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@ -2318,14 +2279,6 @@
        "node": ">=8"
      }
    },
-    "node_modules/for-each": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
-      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
-      "dependencies": {
-        "is-callable": "^1.1.3"
-      }
-    },
    "node_modules/foreground-child": {
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
@ -2395,7 +2348,8 @@
    "node_modules/function-bind": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
+      "peer": true
    },
    "node_modules/gensync": {
      "version": "1.0.0-beta.2",
@ -2414,20 +2368,6 @@
        "node": "6.* || 8.* || >= 10.*"
      }
    },
-    "node_modules/get-intrinsic": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz",
-      "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==",
-      "dependencies": {
-        "function-bind": "^1.1.1",
-        "has": "^1.0.3",
-        "has-proto": "^1.0.1",
-        "has-symbols": "^1.0.3"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/get-package-type": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
@ -2595,17 +2535,6 @@
        "node": ">=0.10.0"
      }
    },
-    "node_modules/gopd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
-      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
-      "dependencies": {
-        "get-intrinsic": "^1.1.3"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/graceful-fs": {
      "version": "4.2.9",
      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz",
@ -2615,6 +2544,7 @@
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+      "peer": true,
      "dependencies": {
        "function-bind": "^1.1.1"
      },
@ -2630,42 +2560,6 @@
        "node": ">=8"
      }
    },
-    "node_modules/has-proto": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
-      "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/has-tostringtag": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
-      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
-      "dependencies": {
-        "has-symbols": "^1.0.2"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/html-escaper": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
@ -2681,11 +2575,6 @@
        "node": ">=10.17.0"
      }
    },
-    "node_modules/immediate": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
-      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
-    },
    "node_modules/import-local": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz",
@ -2737,32 +2626,6 @@
        "validator": "^13.7.0"
      }
    },
-    "node_modules/is-arguments": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
-      "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
-      "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/is-callable": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
-      "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/is-core-module": {
      "version": "2.13.0",
      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz",
@ -2800,20 +2663,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/is-generator-function": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
-      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
-      "dependencies": {
-        "has-tostringtag": "^1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/is-glob": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz",
@ -2864,20 +2713,6 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
-    "node_modules/is-typed-array": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz",
-      "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==",
-      "dependencies": {
-        "which-typed-array": "^1.1.11"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/is-valid-path": {
      "version": "0.1.1",
      "resolved": "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz",
@ -3808,14 +3643,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/lie": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
-      "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
-      "dependencies": {
-        "immediate": "~3.0.5"
-      }
-    },
    "node_modules/lines-and-columns": {
      "version": "1.2.4",
      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@ -3856,14 +3683,6 @@
        "node": ">=4"
      }
    },
-    "node_modules/localforage": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz",
-      "integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==",
-      "dependencies": {
-        "lie": "3.1.1"
-      }
-    },
    "node_modules/locate-path": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
@ -5117,18 +4936,6 @@
        "node": ">= 4.0.0"
      }
    },
-    "node_modules/util": {
-      "version": "0.12.5",
-      "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz",
-      "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==",
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "is-arguments": "^1.0.4",
-        "is-generator-function": "^1.0.7",
-        "is-typed-array": "^1.1.3",
-        "which-typed-array": "^1.1.2"
-      }
-    },
    "node_modules/v8-compile-cache-lib": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
@ -5191,24 +4998,6 @@
        "node": ">= 8"
      }
    },
-    "node_modules/which-typed-array": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz",
-      "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==",
-      "dependencies": {
-        "available-typed-arrays": "^1.0.5",
-        "call-bind": "^1.0.2",
-        "for-each": "^0.3.3",
-        "gopd": "^1.0.1",
-        "has-tostringtag": "^1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/wrap-ansi": {
      "version": "7.0.0",
      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@ -6319,21 +6108,6 @@
      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
      "optional": true
    },
-    "@seald-io/binary-search-tree": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.3.tgz",
-      "integrity": "sha512-qv3jnwoakeax2razYaMsGI/luWdliBLHTdC6jU55hQt1hcFqzauH/HsBollQ7IR4ySTtYhT+xyHoijpA16C+tA=="
-    },
-    "@seald-io/nedb": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/@seald-io/nedb/-/nedb-4.0.2.tgz",
-      "integrity": "sha512-gJ91fT1sgh2cLXYVcTSh7khZ8LdemI8+SojCdpZ5wy+DUQ4fSrEwGqOwbdV49NDs2BBO6GeBpSb8CnhG2IW1rw==",
-      "requires": {
-        "@seald-io/binary-search-tree": "^1.0.3",
-        "localforage": "^1.9.0",
-        "util": "^0.12.4"
-      }
-    },
    "@sinclair/typebox": {
      "version": "0.27.8",
      "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
@ -6608,11 +6382,6 @@
      "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz",
      "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q=="
    },
-    "available-typed-arrays": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
-      "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw=="
-    },
    "babel-jest": {
      "version": "29.6.4",
      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz",
@ -6761,15 +6530,6 @@
      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
      "peer": true
    },
-    "call-bind": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
-      "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
-      "requires": {
-        "function-bind": "^1.1.1",
-        "get-intrinsic": "^1.0.2"
-      }
-    },
    "callsites": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@ -7089,14 +6849,6 @@
        "path-exists": "^4.0.0"
      }
    },
-    "for-each": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
-      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
-      "requires": {
-        "is-callable": "^1.1.3"
-      }
-    },
    "foreground-child": {
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
@ -7145,7 +6897,8 @@
    "function-bind": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
+      "peer": true
    },
    "gensync": {
      "version": "1.0.0-beta.2",
@ -7158,17 +6911,6 @@
      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
    },
-    "get-intrinsic": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz",
-      "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==",
-      "requires": {
-        "function-bind": "^1.1.1",
-        "has": "^1.0.3",
-        "has-proto": "^1.0.1",
-        "has-symbols": "^1.0.3"
-      }
-    },
    "get-package-type": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
@ -7293,14 +7035,6 @@
        }
      }
    },
-    "gopd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
-      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
-      "requires": {
-        "get-intrinsic": "^1.1.3"
-      }
-    },
    "graceful-fs": {
      "version": "4.2.9",
      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz",
@ -7310,6 +7044,7 @@
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+      "peer": true,
      "requires": {
        "function-bind": "^1.1.1"
      }
@ -7319,24 +7054,6 @@
      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
    },
-    "has-proto": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
-      "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg=="
-    },
-    "has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
-    },
-    "has-tostringtag": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
-      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
-      "requires": {
-        "has-symbols": "^1.0.2"
-      }
-    },
    "html-escaper": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
@ -7349,11 +7066,6 @@
      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
      "peer": true
    },
-    "immediate": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
-      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
-    },
    "import-local": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz",
@ -7393,20 +7105,6 @@
        "validator": "^13.7.0"
      }
    },
-    "is-arguments": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
-      "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
-      "requires": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
-      }
-    },
-    "is-callable": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
-      "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA=="
-    },
    "is-core-module": {
      "version": "2.13.0",
      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz",
@ -7432,14 +7130,6 @@
      "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==",
      "peer": true
    },
-    "is-generator-function": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
-      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
-      "requires": {
-        "has-tostringtag": "^1.0.0"
-      }
-    },
    "is-glob": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz",
@ -7472,14 +7162,6 @@
      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
      "peer": true
    },
-    "is-typed-array": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz",
-      "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==",
-      "requires": {
-        "which-typed-array": "^1.1.11"
-      }
-    },
    "is-valid-path": {
      "version": "0.1.1",
      "resolved": "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz",
@ -8186,14 +7868,6 @@
      "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==",
      "peer": true
    },
-    "lie": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
-      "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
-      "requires": {
-        "immediate": "~3.0.5"
-      }
-    },
    "lines-and-columns": {
      "version": "1.2.4",
      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@ -8227,14 +7901,6 @@
        }
      }
    },
-    "localforage": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz",
-      "integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==",
-      "requires": {
-        "lie": "3.1.1"
-      }
-    },
    "locate-path": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
@ -9116,18 +8782,6 @@
      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
      "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
    },
-    "util": {
-      "version": "0.12.5",
-      "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz",
-      "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==",
-      "requires": {
-        "inherits": "^2.0.3",
-        "is-arguments": "^1.0.4",
-        "is-generator-function": "^1.0.7",
-        "is-typed-array": "^1.1.3",
-        "which-typed-array": "^1.1.2"
-      }
-    },
    "v8-compile-cache-lib": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
@ -9180,18 +8834,6 @@
        "isexe": "^2.0.0"
      }
    },
-    "which-typed-array": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz",
-      "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==",
-      "requires": {
-        "available-typed-arrays": "^1.0.5",
-        "call-bind": "^1.0.2",
-        "for-each": "^0.3.3",
-        "gopd": "^1.0.1",
-        "has-tostringtag": "^1.0.0"
-      }
-    },
    "wrap-ansi": {
      "version": "7.0.0",
      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
--- a/package.json
+++ b/package.json
@ -2,11 +2,12 @@
  "name": "iptv",
  "scripts": {
    "act:check": "act pull_request -W .github/workflows/check.yml",
+    "act:format": "act workflow_dispatch -W .github/workflows/format.yml",
    "act:update": "act workflow_dispatch -W .github/workflows/update.yml",
    "api:load": "./scripts/commands/api/load.sh",
    "api:generate": "npm run ts-node scripts/commands/api/generate.ts",
    "api:deploy": "npx gh-pages-clean && npx gh-pages -a -m \"Deploy to iptv-org/api\" -d .api -r https://$GITHUB_TOKEN@github.com/iptv-org/api.git",
-    "db:create": "npm run ts-node scripts/commands/database/create.ts",
+    "playlist:format": "npm run ts-node scripts/commands/playlist/format.ts",
    "playlist:update": "npm run ts-node scripts/commands/playlist/update.ts",
    "playlist:generate": "npm run ts-node scripts/commands/playlist/generate.ts",
    "playlist:validate": "npm run ts-node scripts/commands/playlist/validate.ts",
@ -14,9 +15,9 @@
    "playlist:deploy": "npx gh-pages-clean && npx gh-pages -m \"Deploy to GitHub Pages\" -d .gh-pages -r https://$GITHUB_TOKEN@github.com/iptv-org/iptv.git",
    "readme:update": "npm run ts-node scripts/commands/readme/update.ts",
    "report:create": "npm run ts-node scripts/commands/report/create.ts",
-    "format": "npm run db:create && npm run playlist:format",
+    "format": "npm run api:load && npm run playlist:format",
    "check": "npm run api:load && npm run playlist:lint && npm run playlist:validate",
-    "update": "npm run api:load && npm run db:create && npm run playlist:generate && npm run api:generate && npm run readme:update",
+    "update": "npm run api:load && npm run playlist:generate && npm run api:generate && npm run readme:update",
    "deploy": "npm run playlist:deploy && npm run api:deploy",
    "report": "npm run api:load && npm run report:create",
    "test": "jest --runInBand",
@ -45,7 +46,6 @@
    "@octokit/plugin-paginate-rest": "^7.1.2",
    "@octokit/plugin-rest-endpoint-methods": "^7.1.3",
    "@octokit/types": "^11.1.0",
-    "@seald-io/nedb": "^4.0.2",
    "@types/fs-extra": "^11.0.1",
    "@types/glob": "^8.1.0",
    "@types/jest": "^29.5.4",
--- a/scripts/commands/api/generate.ts
+++ b/scripts/commands/api/generate.ts
@ -1,16 +1,16 @@
-import { API_DIR, DB_DIR } from '../../constants'
-import { Logger, Database, Collection, Storage } from '../../core'
+import { API_DIR, STREAMS_DIR } from '../../constants'
+import { Logger, PlaylistParser, Storage } from '../../core'
 import { Stream } from '../../models'

 async function main() {
  const logger = new Logger()

-  logger.info(`loading streams...`)
-  const db = new Database(DB_DIR)
-  const dbStreams = await db.load('streams.db')
-  const docs = await dbStreams.find({})
-
-  const streams = new Collection(docs as any[])
+  logger.info('loading streams...')
+  const streamsStorage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({ storage: streamsStorage })
+  const files = await streamsStorage.list('**/*.m3u')
+  let streams = await parser.parse(files)
+  streams = streams
    .map(data => new Stream(data))
    .orderBy((stream: Stream) => stream.channel)
    .map((stream: Stream) => stream.toJSON())
@ -18,8 +18,8 @@ async function main() {
  logger.info(`found ${streams.count()} streams`)

  logger.info('saving to .api/streams.json...')
-  const storage = new Storage(API_DIR)
-  await storage.save('streams.json', streams.toJSON())
+  const apiStorage = new Storage(API_DIR)
+  await apiStorage.save('streams.json', streams.toJSON())
 }

 main()
--- a/scripts/commands/api/load.sh
+++ b/scripts/commands/api/load.sh
@ -4,7 +4,6 @@ mkdir -p temp/data
 curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json 
 curl -L -o temp/data/categories.json https://iptv-org.github.io/api/categories.json 
 curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json
-curl -L -o temp/data/streams.json https://iptv-org.github.io/api/streams.json
 curl -L -o temp/data/countries.json https://iptv-org.github.io/api/countries.json 
 curl -L -o temp/data/languages.json https://iptv-org.github.io/api/languages.json 
 curl -L -o temp/data/regions.json https://iptv-org.github.io/api/regions.json 
--- a/scripts/commands/database/create.ts
+++ b/scripts/commands/database/create.ts
@ -1,33 +0,0 @@
-import { Storage, Logger, PlaylistParser, Collection, Database } from '../../core'
-import { Stream, Playlist } from '../../models'
-import { STREAMS_DIR, DB_DIR } from '../../constants'
-
-async function main() {
-  const logger = new Logger()
-
-  logger.info(`looking for streams...`)
-  const storage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({
-    storage
-  })
-  const files = await storage.list(`**/*.m3u`)
-  let streams = new Collection()
-  for (let filepath of files) {
-    const playlist: Playlist = await parser.parse(filepath)
-    streams = streams.concat(playlist.streams)
-  }
-
-  logger.info(`found ${streams.count()} streams`)
-
-  logger.info('clean up the storage...')
-  const dbStorage = new Storage(DB_DIR)
-  await dbStorage.clear('streams.db')
-
-  logger.info('saving streams to the database...')
-  const db = new Database(DB_DIR)
-  const dbStreams = await db.load('streams.db')
-  const data = streams.map((stream: Stream) => stream.data()).all()
-  await dbStreams.insert(data)
-}
-
-main()
--- a/scripts/commands/playlist/format.ts
+++ b/scripts/commands/playlist/format.ts
@ -0,0 +1,67 @@
+import { STREAMS_DIR, DATA_DIR } from '../../constants'
+import { Storage, Logger, PlaylistParser, Collection } from '../../core'
+import { Stream, Playlist, Channel } from '../../models'
+import { program } from 'commander'
+
+program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
+
+async function main() {
+  const storage = new Storage(STREAMS_DIR)
+  const logger = new Logger()
+
+  logger.info('loading channels from api...')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsContent = await dataStorage.json('channels.json')
+  const groupedChannels = new Collection(channelsContent)
+    .map(data => new Channel(data))
+    .keyBy((channel: Channel) => channel.id)
+
+  logger.info('loading streams...')
+  const parser = new PlaylistParser({ storage })
+  const files = program.args.length ? program.args : await storage.list('**/*.m3u')
+  let streams = await parser.parse(files)
+
+  logger.info(`found ${streams.count()} streams`)
+
+  logger.info('normalizing links...')
+  streams = streams.map(stream => {
+    stream.normalizeURL()
+    return stream
+  })
+
+  logger.info('removing duplicates...')
+  streams = streams.uniqBy(stream => stream.url)
+
+  logger.info('removing wrong id...')
+  streams = streams.map((stream: Stream) => {
+    if (groupedChannels.missing(stream.channel)) {
+      stream.channel = ''
+    }
+
+    return stream
+  })
+
+  logger.info('sorting links...')
+  streams = streams.orderBy(
+    [
+      (stream: Stream) => stream.name,
+      (stream: Stream) => parseInt(stream.quality.replace('p', '')),
+      (stream: Stream) => stream.label,
+      (stream: Stream) => stream.url
+    ],
+    ['asc', 'desc', 'asc', 'asc']
+  )
+
+  logger.info('saving...')
+  const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  for (let filepath of groupedStreams.keys()) {
+    const streams = groupedStreams.get(filepath) || []
+
+    if (!streams.length) return
+
+    const playlist = new Playlist(streams, { public: false })
+    await storage.save(filepath, playlist.toString())
+  }
+}
+
+main()
--- a/scripts/commands/playlist/generate.ts
+++ b/scripts/commands/playlist/generate.ts
@ -1,6 +1,5 @@
-import { File, Storage } from '../../core'
+import { File, PlaylistParser, Storage } from '../../core'
 import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
-import { Database } from '../../core/database'
 import { Collection } from '../../core/collection'
 import { Logger } from '../../core/logger'
 import _ from 'lodash'
@ -16,32 +15,31 @@ import {
  IndexLanguageGenerator,
  IndexRegionGenerator
 } from '../../generators'
-import { DATA_DIR, DB_DIR, LOGS_DIR } from '../../constants'
+import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'

 async function main() {
  const logger = new Logger()
+  const dataStorage = new Storage(DATA_DIR)

-  const storage = new Storage(DATA_DIR)
-
-  const channelsContent = await storage.json('channels.json')
+  logger.info('loading data from api...')
+  const channelsContent = await dataStorage.json('channels.json')
  const channels = new Collection(channelsContent).map(data => new Channel(data))
-
-  const categoriesContent = await storage.json('categories.json')
+  const categoriesContent = await dataStorage.json('categories.json')
  const categories = new Collection(categoriesContent).map(data => new Category(data))
-
-  const countriesContent = await storage.json('countries.json')
+  const countriesContent = await dataStorage.json('countries.json')
  const countries = new Collection(countriesContent).map(data => new Country(data))
-
-  const languagesContent = await storage.json('languages.json')
+  const languagesContent = await dataStorage.json('languages.json')
  const languages = new Collection(languagesContent).map(data => new Language(data))
-
-  const regionsContent = await storage.json('regions.json')
+  const regionsContent = await dataStorage.json('regions.json')
  const regions = new Collection(regionsContent).map(data => new Region(data))
-
-  const subdivisionsContent = await storage.json('subdivisions.json')
+  const subdivisionsContent = await dataStorage.json('subdivisions.json')
  const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))

-  const streams = await loadStreams({ channels, categories, languages })
+  logger.info('loading streams...')
+  let streams = await loadStreams({ channels, categories, languages })
+  let totalStreams = streams.count()
+  streams = streams.uniqBy((stream: Stream) => stream.channel || _.uniqueId())
+  logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`)

  const generatorsLogger = new Logger({
    stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
@ -49,7 +47,6 @@ async function main() {

  logger.info('generating categories/...')
  await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
-
  logger.info('generating countries/...')
  await new CountriesGenerator({
    countries,
@ -58,10 +55,8 @@ async function main() {
    subdivisions,
    logger: generatorsLogger
  }).generate()
-
  logger.info('generating languages/...')
  await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
-
  logger.info('generating regions/...')
  await new RegionsGenerator({
    streams,
@ -69,16 +64,12 @@ async function main() {
    subdivisions,
    logger: generatorsLogger
  }).generate()
-
  logger.info('generating index.m3u...')
  await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
-
  logger.info('generating index.nsfw.m3u...')
  await new IndexNsfwGenerator({ streams, logger: generatorsLogger }).generate()
-
  logger.info('generating index.category.m3u...')
  await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
-
  logger.info('generating index.country.m3u...')
  await new IndexCountryGenerator({
    streams,
@ -87,10 +78,8 @@ async function main() {
    subdivisions,
    logger: generatorsLogger
  }).generate()
-
  logger.info('generating index.language.m3u...')
  await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
-
  logger.info('generating index.region.m3u...')
  await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
 }
@ -110,13 +99,13 @@ async function loadStreams({
  const groupedCategories = categories.keyBy(category => category.id)
  const groupedLanguages = languages.keyBy(language => language.code)

-  const db = new Database(DB_DIR)
-  const dbStreams = await db.load('streams.db')
-  const docs = await dbStreams.find({})
-  const streams = new Collection(docs as any[])
-    .map((data: any) => new Stream(data))
+  const storage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({ storage })
+  const files = await storage.list('**/*.m3u')
+  let streams = await parser.parse(files)
+
+  streams = streams
    .orderBy([(stream: Stream) => stream.channel, (stream: Stream) => stream.url], ['asc', 'asc'])
-    .uniqBy((stream: Stream) => stream.channel || _.uniqueId())
    .map((stream: Stream) => {
      const channel: Channel | undefined = groupedChannels.get(stream.channel)

--- a/scripts/commands/playlist/update.ts
+++ b/scripts/commands/playlist/update.ts
@ -1,6 +1,6 @@
-import { DB_DIR, DATA_DIR, STREAMS_DIR } from '../../constants'
-import { Database, Storage, Logger, Collection, Dictionary, IssueLoader } from '../../core'
-import { Stream, Playlist, Channel } from '../../models'
+import { DATA_DIR, STREAMS_DIR } from '../../constants'
+import { Storage, Logger, Collection, Dictionary, IssueLoader, PlaylistParser } from '../../core'
+import { Stream, Playlist, Channel, Issue } from '../../models'

 let processedIssues = new Collection()
 let streams: Collection
@ -10,19 +10,19 @@ async function main() {
  const logger = new Logger({ disabled: true })
  const loader = new IssueLoader()

-  logger.info('loading streams...')
-  const db = new Database(DB_DIR)
-  const docs = await db.load('streams.db')
-  const dbStreams = await docs.find({})
-
-  streams = new Collection(dbStreams as any[]).map(data => new Stream(data))
-
-  const storage = new Storage(DATA_DIR)
-  const channelsContent = await storage.json('channels.json')
+  logger.info('loading channels from api...')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsContent = await dataStorage.json('channels.json')
  groupedChannels = new Collection(channelsContent)
    .map(data => new Channel(data))
    .keyBy((channel: Channel) => channel.id)

+  logger.info('loading streams...')
+  const streamsStorage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({ storage: streamsStorage })
+  const files = await streamsStorage.list('**/*.m3u')
+  streams = await parser.parse(files)
+
  logger.info('removing broken streams...')
  await removeStreams(loader)

@ -32,25 +32,7 @@ async function main() {
  logger.info('add new streams...')
  await addStreams(loader)

-  logger.info('normalizing links...')
-  streams = streams.map(stream => {
-    stream.normalizeURL()
-    return stream
-  })
-
-  logger.info('sorting links...')
-  streams = streams.orderBy(
-    [
-      (stream: Stream) => stream.name,
-      (stream: Stream) => parseInt(stream.quality.replace('p', '')),
-      (stream: Stream) => stream.label,
-      (stream: Stream) => stream.url
-    ],
-    ['asc', 'desc', 'asc', 'asc']
-  )
-
  logger.info('saving...')
-  const streamsStorage = new Storage(STREAMS_DIR)
  const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
  for (let filepath of groupedStreams.keys()) {
    const streams = groupedStreams.get(filepath) || []
@ -69,19 +51,22 @@ main()

 async function removeStreams(loader: IssueLoader) {
  const issues = await loader.load({ labels: ['streams:remove', 'approved'] })
-  issues.forEach((data: Dictionary) => {
+  issues.forEach((issue: Issue) => {
+    const data = issue.data
    if (data.missing('stream_url')) return

    const removed = streams.remove((_stream: Stream) => _stream.url === data.get('stream_url'))
    if (removed.notEmpty()) {
-      processedIssues.add(data.get('issue_number'))
+      processedIssues.add(issue.number)
    }
  })
 }

 async function editStreams(loader: IssueLoader) {
  const issues = await loader.load({ labels: ['streams:edit', 'approved'] })
-  issues.forEach((data: Dictionary) => {
+  issues.forEach((issue: Issue) => {
+    const data = issue.data
+
    if (data.missing('stream_url')) return

    let stream = streams.first(
@ -111,13 +96,14 @@ async function editStreams(loader: IssueLoader) {
    streams.remove((_stream: Stream) => _stream.channel === stream.channel)
    streams.add(stream)

-    processedIssues.add(data.get('issue_number'))
+    processedIssues.add(issue.number)
  })
 }

 async function addStreams(loader: IssueLoader) {
  const issues = await loader.load({ labels: ['streams:add', 'approved'] })
-  issues.forEach((data: Dictionary) => {
+  issues.forEach((issue: Issue) => {
+    const data = issue.data
    if (data.missing('channel_id') || data.missing('stream_url')) return
    if (streams.includes((_stream: Stream) => _stream.url === data.get('stream_url'))) return

@ -138,6 +124,6 @@ async function addStreams(loader: IssueLoader) {
    })

    streams.add(stream)
-    processedIssues.add(data.get('issue_number'))
+    processedIssues.add(issue.number)
  })
 }
--- a/scripts/commands/playlist/validate.ts
+++ b/scripts/commands/playlist/validate.ts
@ -5,7 +5,6 @@ import chalk from 'chalk'
 import { transliterate } from 'transliteration'
 import _ from 'lodash'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
-import path from 'path'

 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)

@ -19,73 +18,70 @@ async function main() {
  const logger = new Logger()

  logger.info(`loading blocklist...`)
-  const storage = new Storage(DATA_DIR)
-  const channelsContent = await storage.json('channels.json')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsContent = await dataStorage.json('channels.json')
  const channels = new Collection(channelsContent).map(data => new Channel(data))
-  const blocklistContent = await storage.json('blocklist.json')
+  const blocklistContent = await dataStorage.json('blocklist.json')
  const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))

  logger.info(`found ${blocklist.count()} records`)

-  let errors = new Collection()
-  let warnings = new Collection()
+  logger.info('loading streams...')
  const streamsStorage = new Storage(STREAMS_DIR)
  const parser = new PlaylistParser({ storage: streamsStorage })
  const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
-  for (const filepath of files) {
-    const file = new File(filepath)
-    if (file.extension() !== 'm3u') continue
+  const streams = await parser.parse(files)

+  logger.info(`found ${streams.count()} streams`)
+
+  let errors = new Collection()
+  let warnings = new Collection()
+  let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  for (const filepath of groupedStreams.keys()) {
+    const streams = groupedStreams.get(filepath)
+    if (!streams) continue
+
+    const file = new File(filepath)
    const [, countryCode] = file.basename().match(/([a-z]{2})(|_.*)\.m3u/i) || [null, '']

    const log = new Collection()
    const buffer = new Dictionary()
-    try {
-      const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
-      const playlist = await parser.parse(relativeFilepath)
-      playlist.streams.forEach((stream: Stream) => {
-        const channelNotInDatabase =
-          stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
-        if (channelNotInDatabase) {
-          log.add({
-            type: 'warning',
-            line: stream.line,
-            message: `"${stream.channel}" is not in the database`
-          })
-        }
-
-        const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
-        if (alreadyOnPlaylist) {
-          log.add({
-            type: 'warning',
-            line: stream.line,
-            message: `"${stream.url}" is already on the playlist`
-          })
-        } else {
-          buffer.set(stream.url, true)
-        }
-
-        const channelId = generateChannelId(stream.name, countryCode)
-        const blocked = blocklist.first(
-          blocked =>
-            stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
-            channelId.toLowerCase() === blocked.channel.toLowerCase()
-        )
-        if (blocked) {
-          log.add({
-            type: 'error',
-            line: stream.line,
-            message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
-          })
-        }
-      })
-    } catch (error) {
-      log.add({
-        type: 'error',
-        line: 0,
-        message: error.message.toLowerCase()
-      })
-    }
+    streams.forEach((stream: Stream) => {
+      const channelNotInDatabase =
+        stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
+      if (channelNotInDatabase) {
+        log.add({
+          type: 'warning',
+          line: stream.line,
+          message: `"${stream.channel}" is not in the database`
+        })
+      }
+
+      const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
+      if (alreadyOnPlaylist) {
+        log.add({
+          type: 'warning',
+          line: stream.line,
+          message: `"${stream.url}" is already on the playlist`
+        })
+      } else {
+        buffer.set(stream.url, true)
+      }
+
+      const channelId = generateChannelId(stream.name, countryCode)
+      const blocked = blocklist.first(
+        blocked =>
+          stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
+          channelId.toLowerCase() === blocked.channel.toLowerCase()
+      )
+      if (blocked) {
+        log.add({
+          type: 'error',
+          line: stream.line,
+          message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
+        })
+      }
+    })

    if (log.notEmpty()) {
      logger.info(`\n${chalk.underline(filepath)}`)
--- a/scripts/commands/report/create.ts
+++ b/scripts/commands/report/create.ts
@ -1,36 +1,43 @@
-import { DATA_DIR } from '../../constants'
-import { Collection, Dictionary, IssueLoader, Storage } from '../../core'
-import { Blocked, Channel, Stream } from '../../models'
+import { DATA_DIR, STREAMS_DIR } from '../../constants'
+import { Collection, Dictionary, IssueLoader, Storage, Logger, PlaylistParser } from '../../core'
+import { Blocked, Channel, Issue, Stream } from '../../models'

 async function main() {
+  const logger = new Logger()
  const loader = new IssueLoader()

  const storage = new Storage(DATA_DIR)

+  logger.info('loading channels from api...')
  const channelsContent = await storage.json('channels.json')
  const groupedChannels = new Collection(channelsContent)
    .map(data => new Channel(data))
    .groupBy((channel: Channel) => channel.id)

-  const streamsContent = await storage.json('streams.json')
-  const groupedStreams = new Collection(streamsContent)
-    .map(data => new Stream(data))
-    .groupBy((stream: Stream) => stream.url)
-
+  logger.info('loading blocklist from api...')
  const blocklistContent = await storage.json('blocklist.json')
  const groupedBlocklist = new Collection(blocklistContent)
    .map(data => new Blocked(data))
    .groupBy((blocked: Blocked) => blocked.channel)

+  logger.info('loading streams...')
+  const streamsStorage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({ storage: streamsStorage })
+  const files = await streamsStorage.list('**/*.m3u')
+  const streams = await parser.parse(files)
+  const groupedStreams = streams.groupBy((stream: Stream) => stream.url)
+
+  logger.info('loading issue from github...')
  const issues = await loader.load({ labels: ['streams:add'] })

+  logger.info('creating report...')
  const buffer = new Dictionary()
-  const report = issues.map(data => {
-    const channelId = data.get('channel_id') || undefined
-    const streamUrl = data.get('stream_url') || undefined
+  const report = issues.map((issue: Issue) => {
+    const channelId = issue.data.get('channel_id') || undefined
+    const streamUrl = issue.data.get('stream_url') || undefined

    const result = new Dictionary({
-      issueNumber: data.get('issue_number'),
+      issueNumber: issue.number,
      channelId,
      status: undefined
    })
--- a/scripts/constants.ts
+++ b/scripts/constants.ts
@ -5,7 +5,6 @@ export const README_DIR = process.env.README_DIR || './.readme'
 export const API_DIR = process.env.API_DIR || './.api'
 export const DATA_DIR = process.env.DATA_DIR || './temp/data'
 export const LOGS_DIR = process.env.LOGS_DIR || './temp/logs'
-export const DB_DIR = process.env.DB_DIR || './temp/database'
 export const TESTING = process.env.NODE_ENV === 'test' ? true : false
 export const OWNER = 'iptv-org'
 export const REPO = 'iptv'
--- a/scripts/core/database.ts
+++ b/scripts/core/database.ts
@ -1,22 +0,0 @@
-import Datastore from '@seald-io/nedb'
-import * as path from 'path'
-
-export class Database {
-  rootDir: string
-
-  constructor(rootDir: string) {
-    this.rootDir = rootDir
-  }
-
-  async load(filepath: string) {
-    const absFilepath = path.join(this.rootDir, filepath)
-
-    return new Datastore({
-      filename: path.resolve(absFilepath),
-      autoload: true,
-      onload: (error: Error): any => {
-        if (error) console.error(error.message)
-      }
-    })
-  }
-}
--- a/scripts/core/index.ts
+++ b/scripts/core/index.ts
@ -1,4 +1,3 @@
-export * from './database'
 export * from './logger'
 export * from './playlistParser'
 export * from './numberParser'
--- a/scripts/core/issueParser.ts
+++ b/scripts/core/issueParser.ts
@ -1,33 +1,31 @@
 import { Dictionary } from './'
+import { Issue } from '../models'
+import _ from 'lodash'
+
+const FIELDS = new Dictionary({
+  'Channel ID': 'channel_id',
+  'Channel ID (required)': 'channel_id',
+  'Broken Link': 'stream_url',
+  'Stream URL': 'stream_url',
+  'Stream URL (optional)': 'stream_url',
+  'Stream URL (required)': 'stream_url',
+  Label: 'label',
+  Quality: 'quality',
+  'Channel Name': 'channel_name',
+  'HTTP User-Agent': 'user_agent',
+  'HTTP Referrer': 'http_referrer',
+  Reason: 'reason',
+  'What happened to the stream?': 'reason',
+  'Possible Replacement (optional)': 'possible_replacement',
+  Notes: 'notes',
+  'Notes (optional)': 'notes'
+})

 export class IssueParser {
-  parse(issue: any): Dictionary {
-    const data = new Dictionary()
-    data.set('issue_number', issue.number)
-
-    const idDict = new Dictionary({
-      'Channel ID': 'channel_id',
-      'Channel ID (required)': 'channel_id',
-      'Broken Link': 'stream_url',
-      'Stream URL': 'stream_url',
-      'Stream URL (optional)': 'stream_url',
-      'Stream URL (required)': 'stream_url',
-      Label: 'label',
-      Quality: 'quality',
-      'Channel Name': 'channel_name',
-      'HTTP User-Agent': 'user_agent',
-      'HTTP Referrer': 'http_referrer',
-      Reason: 'reason',
-      'What happened to the stream?': 'reason',
-      'Possible Replacement (optional)': 'possible_replacement',
-      Notes: 'notes',
-      'Notes (optional)': 'notes'
-    })
-
+  parse(issue: any): Issue {
    const fields = issue.body.split('###')

-    if (!fields.length) return data
-
+    const data = new Dictionary()
    fields.forEach((field: string) => {
      let [_label, , _value] = field.split(/\r?\n/)
      _label = _label ? _label.trim() : ''
@ -35,7 +33,7 @@ export class IssueParser {

      if (!_label || !_value) return data

-      const id: string = idDict.get(_label)
+      const id: string = FIELDS.get(_label)
      const value: string = _value === '_No response_' || _value === 'None' ? '' : _value

      if (!id) return
@ -43,6 +41,6 @@ export class IssueParser {
      data.set(id, value)
    })

-    return data
+    return new Issue({ number: issue.number, data })
  }
 }
--- a/scripts/core/playlistParser.ts
+++ b/scripts/core/playlistParser.ts
@ -1,6 +1,8 @@
 import parser from 'iptv-playlist-parser'
-import { Playlist, Stream } from '../models'
+import { Stream } from '../models'
 import { Collection, Storage } from './'
+import path from 'path'
+import { STREAMS_DIR } from '../constants'

 export class PlaylistParser {
  storage: Storage
@ -9,7 +11,19 @@ export class PlaylistParser {
    this.storage = storage
  }

-  async parse(filepath: string): Promise<Playlist> {
+  async parse(files: string[]): Promise<Collection> {
+    let streams = new Collection()
+
+    for (let filepath of files) {
+      const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
+      const _streams: Collection = await this.parseFile(relativeFilepath)
+      streams = streams.concat(_streams)
+    }
+
+    return streams
+  }
+
+  async parseFile(filepath: string): Promise<Collection> {
    const streams = new Collection()

    const content = await this.storage.read(filepath)
@ -32,7 +46,7 @@ export class PlaylistParser {
      streams.add(stream)
    })

-    return new Playlist(streams)
+    return streams
  }
 }

--- a/scripts/core/storage.ts
+++ b/scripts/core/storage.ts
@ -10,10 +10,12 @@ export class Storage {
    this.rootDir = path.normalize(rootDir || './')
  }

-  list(pattern: string): Promise<string[]> {
-    return glob(pattern, {
+  async list(pattern: string): Promise<string[]> {
+    const files = await glob(pattern, {
      cwd: this.rootDir
    })
+
+    return files.sort()
  }

  async createDir(dir: string): Promise<void> {
--- a/scripts/models/index.ts
+++ b/scripts/models/index.ts
@ -1,3 +1,4 @@
+export * from './issue'
 export * from './playlist'
 export * from './blocked'
 export * from './stream'
--- a/scripts/models/issue.ts
+++ b/scripts/models/issue.ts
@ -0,0 +1,16 @@
+import { Dictionary } from '../core'
+
+type IssueProps = {
+  number: number
+  data: Dictionary
+}
+
+export class Issue {
+  number: number
+  data: Dictionary
+
+  constructor({ number, data }: IssueProps) {
+    this.number = number
+    this.data = data
+  }
+}
--- a/tests/data/expected/.api/streams.json
+++ b/tests/data/expected/.api/streams.json
@ -1 +1 @@
-[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null},{"channel":"BBCNewsHD.uk","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null}]
+[{"channel":"","url":"http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"},{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8","http_referrer":null,"user_agent":null},{"channel":"MeteoMedia.ca","url":"http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8","http_referrer":null,"user_agent":null},{"channel":"VisitXTV.nl","url":"https://stream.visit-x.tv/vxtv/ngrp:live_all/30fps.m3u8","http_referrer":null,"user_agent":null},{"channel":"Zoo.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/zoo","http_referrer":null,"user_agent":null}]
--- a/tests/data/expected/.gh-pages/countries/uk.m3u
+++ b/tests/data/expected/.gh-pages/countries/uk.m3u
@ -1,5 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
--- a/tests/data/expected/.gh-pages/index.country.m3u
+++ b/tests/data/expected/.gh-pages/index.country.m3u
@ -9,9 +9,9 @@ http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8
 http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8
 #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Russia",ЛДПР ТВ (1080p)
 http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="United Kingdom" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
+#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="International",BBC News HD
+http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8
+#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
 #EXTVLCOPT:http-referrer=http://imn.iq
 #EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
 http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
-#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="International",BBC News HD
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8
--- a/tests/data/expected/.gh-pages/index.region.m3u
+++ b/tests/data/expected/.gh-pages/index.region.m3u
@ -9,20 +9,12 @@ http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8
 http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8
 #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Commonwealth of Independent States",ЛДПР ТВ (1080p)
 http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Europe" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Europe",ATV
 https://iptv-all.lanesh4d0w.repl.co/andorra/atv
 #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Europe",Zoo (720p)
 https://iptv-all.lanesh4d0w.repl.co/andorra/zoo
 #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Europe",ЛДПР ТВ (1080p)
 http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Europe, the Middle East and Africa" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Europe, the Middle East and Africa",ATV
 https://iptv-all.lanesh4d0w.repl.co/andorra/atv
 #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Europe, the Middle East and Africa",Zoo (720p)
@ -35,10 +27,6 @@ http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8
 http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8
 #EXTINF:-1 tvg-id="" tvg-logo="" group-title="South Asia",Daawah TV
 http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Worldwide" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Worldwide",ATV
 https://iptv-all.lanesh4d0w.repl.co/andorra/atv
 #EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="Worldwide",BBC News HD
@ -51,3 +39,7 @@ http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8
 https://iptv-all.lanesh4d0w.repl.co/andorra/zoo
 #EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="Worldwide",ЛДПР ТВ (1080p)
 http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8
+#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
+#EXTVLCOPT:http-referrer=http://imn.iq
+#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
+http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
--- a/tests/data/expected/.gh-pages/regions/emea.m3u
+++ b/tests/data/expected/.gh-pages/regions/emea.m3u
@ -1,8 +1,4 @@
 #EXTM3U
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Undefined",ATV
 https://iptv-all.lanesh4d0w.repl.co/andorra/atv
 #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Undefined",Zoo (720p)
--- a/tests/data/expected/.gh-pages/regions/eur.m3u
+++ b/tests/data/expected/.gh-pages/regions/eur.m3u
@ -1,8 +1,4 @@
 #EXTM3U
-#EXTINF:-1 tvg-id="" tvg-logo="" group-title="Undefined" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
 #EXTINF:-1 tvg-id="AndorraTV.ad" tvg-logo="" group-title="Undefined",ATV
 https://iptv-all.lanesh4d0w.repl.co/andorra/atv
 #EXTINF:-1 tvg-id="Zoo.ad" tvg-logo="" group-title="Undefined",Zoo (720p)
--- a/tests/data/expected/database/db_create.streams.db
+++ b/tests/data/expected/database/db_create.streams.db
@ -1,7 +0,0 @@
-{"line":2,"channel":"ATV.ad","quality":"720p","label":"Offline","name":"ATV","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"k4XpZHQAyqyTbcf0"}
-{"line":2,"channel":"LibyasChannel.ly","quality":"","label":"","name":"Libyas Channel","filepath":"ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","httpReferrer":"","userAgent":"","_id":"ki4YjAoNNoIY8sSm"}
-{"line":2,"channel":"","quality":"720p","label":"","name":"1A Network","filepath":"unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","httpReferrer":"","userAgent":"","_id":"IZpCJjjWPaBYh7Dr"}
-{"line":2,"channel":"","quality":"720p","label":"","name":"Fox Sports 2 Asia (Thai)","filepath":"us_blocked.m3u","url":"https://example.com/playlist.m3u8","httpReferrer":"","userAgent":"","_id":"6c4J4vs8K69wMJ7S"}
-{"line":4,"channel":"","quality":"","label":"","name":"TVN","filepath":"us_blocked.m3u","url":"https://example.com/playlist2.m3u8","httpReferrer":"","userAgent":"","_id":"xEmbX384v3t3F5Wg"}
-{"line":6,"channel":"EverydayHeroes.us","quality":"720p","label":"","name":"Everyday Heroes","filepath":"us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","httpReferrer":"","userAgent":"","_id":"BZsRPt8VS4kIJnfi"}
-{"line":2,"channel":"qib22lAq1L.us","quality":"720p","label":"","name":"ABC","filepath":"wrong_id.m3u","url":"https://example.com/playlist2.m3u8","httpReferrer":"","userAgent":"","_id":"eFUlUnST5zJSBWAF"}
--- a/tests/data/expected/logs/generators.log
+++ b/tests/data/expected/logs/generators.log
@ -32,14 +32,13 @@
 {"filepath":"subdivisions/ca-on.m3u","count":1}
 {"filepath":"countries/in.m3u","count":1}
 {"filepath":"countries/ru.m3u","count":1}
-{"filepath":"countries/uk.m3u","count":1}
 {"filepath":"countries/int.m3u","count":1}
 {"filepath":"index.category.m3u","count":8}
 {"filepath":"index.country.m3u","count":7}
 {"filepath":"index.language.m3u","count":7}
 {"filepath":"index.m3u","count":7}
 {"filepath":"index.nsfw.m3u","count":8}
-{"filepath":"index.region.m3u","count":23}
+{"filepath":"index.region.m3u","count":21}
 {"filepath":"languages/eng.m3u","count":1}
 {"filepath":"languages/rus.m3u","count":1}
 {"filepath":"languages/cat.m3u","count":1}
@ -54,8 +53,8 @@
 {"filepath":"regions/cas.m3u","count":0}
 {"filepath":"regions/cenamer.m3u","count":0}
 {"filepath":"regions/cis.m3u","count":1}
-{"filepath":"regions/emea.m3u","count":4}
-{"filepath":"regions/eur.m3u","count":4}
+{"filepath":"regions/emea.m3u","count":3}
+{"filepath":"regions/eur.m3u","count":3}
 {"filepath":"regions/hispam.m3u","count":0}
 {"filepath":"regions/lac.m3u","count":0}
 {"filepath":"regions/latam.m3u","count":0}
--- a/tests/data/expected/streams/ad.m3u
+++ b/tests/data/expected/streams/ad.m3u
@ -1,5 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="AndorraTV.ad" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",ATV (720p) [Offline]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-https://iptv-all.lanesh4d0w.repl.co/andorra/atv
--- a/tests/data/expected/streams/af.m3u
+++ b/tests/data/expected/streams/af.m3u
@ -1,5 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="KayhanTV.af",Kayhan TV
-http://208.93.117.113/live/Stream1/playlist.m3u8
-#EXTINF:-1 tvg-id="Sharq.af",Sharq
-http://51.210.199.50/hls/stream.m3u8
--- a/tests/data/expected/streams/aw.m3u
+++ b/tests/data/expected/streams/aw.m3u
@ -1,6 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="Telearuba.aw",Telearuba (720p)
-http://cdn.setar.aw:1935/Telearuba/smil:telearuba.smil/playlist.m3u8
-#EXTINF:-1 tvg-id="Telearuba.aw" user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",Telearuba (480p) [Not 24/7]
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36
-https://backend-server-dot-telearuba-app.appspot.com/media/livestream13/playlist.m3u8
--- a/tests/data/expected/streams/es.m3u
+++ b/tests/data/expected/streams/es.m3u
@ -1,7 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="",Caillou
-https://dhx-caillou-1-es.samsung.wurl.tv/playlist.m3u8
-#EXTINF:-1 tvg-id="",iHola Play
-https://rakuten-hola-2-es.samsung.wurl.tv/playlist.m3u8
-#EXTINF:-1 tvg-id="",Planeta Junior TV
-https://deaplaneta-planetakidz-1-es.samsung.wurl.tv/playlist.m3u8
--- a/tests/data/expected/streams/ru.m3u
+++ b/tests/data/expected/streams/ru.m3u
@ -1,9 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
-http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8
-#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
-http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8
-#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
-http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8
-#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
-https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?advertisingId=&appName=web&deviceId=5ca525b650be2571e3943c63
--- a/tests/data/expected/streams_format/in.m3u
+++ b/tests/data/expected/streams_format/in.m3u
@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="",Manorama News
+https://ythls.onrender.com/channel/UCP0uG-mcMImgKnJz-VjJZmQ.m3u8
--- a/tests/data/expected/streams_format/nl.m3u
+++ b/tests/data/expected/streams_format/nl.m3u
@ -1,8 +1,4 @@
 #EXTM3U
-#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (1080p) [Geo-blocked]
-http://stream.tvtap.net:8081/live/nl-npo1.stream/30fps.m3u8
-#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (1080p) [Geo-blocked]
-http://stream.tvtap.net:8081/live/nl-npo1.stream/60fps.m3u8
 #EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (342p) [Geo-blocked]
 http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8
 #EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (342p)
--- a/tests/data/expected/streams_update/in.m3u
+++ b/tests/data/expected/streams_update/in.m3u
--- a/tests/data/expected/streams_update/my.m3u
+++ b/tests/data/expected/streams_update/my.m3u
--- a/tests/data/expected/streams_update/uk.m3u
+++ b/tests/data/expected/streams_update/uk.m3u
--- a/tests/data/expected/streams_update/us.m3u
+++ b/tests/data/expected/streams_update/us.m3u
--- a/tests/data/input/data/streams.json
+++ b/tests/data/input/data/streams.json
@ -1 +0,0 @@
-[{"channel":"TUTV.us","url":"https://livestream.telvue.com/templeuni1/f7b44cfafd5c52223d5498196c8a2e7b.sdp/playlist.m3u8","http_referrer":null,"user_agent":null}]
--- a/tests/data/input/database/api_generate.streams.db
+++ b/tests/data/input/database/api_generate.streams.db
@ -1,7 +0,0 @@
-{"line": 2,"quality":null,"label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
-{"line": 2,"quality":null,"label":null,"name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
-{"line": 2,"quality":null,"label":null,"name":"ATV","channel":"AndorraTV.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error"}
-{"line": 2,"quality":null,"label":null,"name":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
-{"line": 2,"quality":null,"label":null,"name":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
-{"line": 2,"quality":null,"label":null,"name":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}
-{"line": 2,"quality":null,"label":null,"name":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","httpReferrer":null,"userAgent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
--- a/tests/data/input/database/playlist_generate.streams.db
+++ b/tests/data/input/database/playlist_generate.streams.db
@ -1,14 +0,0 @@
-{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF0"}
-{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/timeout.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF1"}
-{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF2"}
-{"line": 2,"quality":"1080p","label":null,"name":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/error.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF3"}
-{"line": 2,"quality":"720p","label":"Geo-blocked","name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"3TbieV1ptnZVCId5"}
-{"line": 2,"quality":null,"label":null,"name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","httpReferrer":null,"userAgent":null,"_id":"3TbieV1ptnZVCIdn"}
-{"line": 2,"quality":null,"label":null,"name":"ATV","channel":"AndorraTV.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":null,"userAgent":null,"_id":"I6cjG2xCBRFFP44z"}
-{"line": 2,"quality":"720p","label":"Not 24/7","name":"Andorra TV","channel":"","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"WTbieV1ptnZVCIdn"}
-{"line": 2,"quality":null,"label":null,"name":"Visit-X TV","channel":"VisitXTV.nl","filepath":"nl.m3u","url":"https://stream.visit-x.tv/vxtv/ngrp:live_all/30fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF5"}
-{"line": 2,"quality":null,"label":null,"name":"Visit-X TV","channel":"VisitXTV.nl","filepath":"nl.m3u","url":"https://stream.visit-x.tv/vxtv/ngrp:live_all/60fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF6"}
-{"line": 2,"quality":null,"label":null,"name":"Daawah TV","channel":"","filepath":"in.m3u","url":"http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF9"}
-{"line": 2,"quality":null,"label":null,"name":"Meteomedia","channel":"MeteoMedia.ca","filepath":"in.m3u","url":"http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgP49"}
-{"line": 2,"quality":"480p","label":null,"name":"Zoo","channel":"Zoo.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/zoo?480","httpReferrer":null,"userAgent":null,"_id":"I6cjG2xCBRFFP4s3"}
-{"line": 2,"quality":"720p","label":null,"name":"Zoo","channel":"Zoo.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/zoo","httpReferrer":null,"userAgent":null,"_id":"I6cjG2xCBRFFP4sz"}
--- a/tests/data/input/database/playlist_update.streams.db
+++ b/tests/data/input/database/playlist_update.streams.db
@ -1,21 +0,0 @@
-{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"https://service-stitcher.clusters.pluto.tv/stitch/hls/channel/5ca525b650be2571e3943c63/master.m3u8?deviceId=5ca525b650be2571e3943c63&appName=web&advertisingId=","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF3"}
-{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF0"}
-{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF1"}
-{"line": 2,"name":"ЛДПР ТВ","quality":"1080p","label":null,"channel":"LDPRTV.ru","filepath":"ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2ST8btby3mmsgPF2"}
-{"line": 2,"name":"BBC News HD","quality":"720p","label":"Not 24/7","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","httpReferrer":null,"userAgent":null,"_id":"3TbieV1ptnZVCIdn"}
-{"line": 2,"quality":"720p","label":"Offline","name":"ATV","channel":"AndorraTV.ad","filepath":"ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","httpReferrer":"http://imn.iq","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"I6cjG2xCBRFFP4sz"}
-{"line": 2,"quality":"480p","label":"Geo-blocked","name":"BBC News HD","channel":"BBCNews.uk","filepath":"uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"WTbieV1ptnZVCIdn"}
-{"line": 2,"quality":null,"label":null,"name":"Kayhan TV","channel":"KayhanTV.af","filepath":"af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"cFFpFVzSn6xFMUF3"}
-{"line": 2,"quality":null,"label":null,"name":"Sharq","channel":"Sharq.af","filepath":"af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","httpReferrer":null,"userAgent":null,"_id":"u7iyA6cjtf1iWWAZ"}
-{"line": 2,"quality":"342p","label":"Geo-blocked","channel":"NPO1.nl","name":"NPO 1","filepath":"nl.m3u","url":"http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8","httpReferrer":null,"userAgent":null,"_id":"mvUyDVuS5gc8gLJV"}
-{"line": 2,"quality":"1080p","label":"Geo-blocked","channel":"NPO1.nl","name":"NPO 1","filepath":"nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo1.stream/30fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"8WVbsxsYeOL7kHQl"}
-{"line": 2,"quality":"1080p","label":"Geo-blocked","channel":"NPO1.nl","name":"NPO 1","filepath":"nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo1.stream/60fps.m3u8","httpReferrer":null,"userAgent":null,"_id":"8WVbsxsYeOL7kHQB"}
-{"line": 2,"quality":"342p","label":null,"channel":"NPO2.nl","name":"NPO 2","filepath":"nl.m3u","url":"http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8","httpReferrer":null,"userAgent":null,"_id":"2p1TNGO0mF0MJOGy"}
-{"line": 2,"quality":"302p","label":"Geo-blocked","channel":"NPO2.nl","name":"NPO 2","filepath":"nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo2.stream/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"nhL85BL7YM5OR7cn"}
-{"line": 2,"quality":null,"label":null,"name":"Tele 2000","channel":"Tele2000.pe","filepath":"pe.m3u","url":"https://servilive.com:3126/live/tele2000live.m3u8","httpReferrer":"https://example2.com/","userAgent":null,"_id":"cF0pFVzSn6xFMUF3"}
-{"line": 2,"quality":null,"label":null,"name":"Planeta Junior TV","channel":"","filepath":"es.m3u","url":"https://deaplaneta-planetakidz-1-es.samsung.wurl.tv/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"1BT8btby3mmsgPF0"}
-{"line": 2,"quality":null,"label":null,"name":"Caillou","channel":"","filepath":"es.m3u","url":"https://dhx-caillou-1-es.samsung.wurl.tv/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"3BT8btby3mmsgPF0"}
-{"line": 2,"quality":null,"label":null,"name":"iHola Play","channel":"","filepath":"es.m3u","url":"https://rakuten-hola-2-es.samsung.wurl.tv/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"2BT8btby3mmsgPF0"}
-{"line": 2,"quality":"720p","label":null,"name":"Telearuba","channel":"Telearuba.aw","filepath":"aw.m3u","url":"http://cdn.setar.aw:1935/Telearuba/smil:telearuba.smil/playlist.m3u8","httpReferrer":null,"userAgent":null,"_id":"6BT8btby3mmsgPF0"}
-{"line": 2,"quality":"480p","label":"Not 24/7","name":"Telearuba","channel":"Telearuba.aw","filepath":"aw.m3u","url":"https://backend-server-dot-telearuba-app.appspot.com/media/livestream13/playlist.m3u8","httpReferrer":null,"userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36","_id":"4BT8btby3mmsgPF0"}
-{"line": 2,"quality":"","label":"","name":"Telearuba","channel":"","filepath":"bg.m3u","url":"https://ythls.onrender.com/channel/UC40TUSUx490U5uR1lZt3Ajgm3u8","httpReferrer":null,"userAgent":null,"_id":"4BT8btby3mmsgSF0"}
--- a/tests/data/input/streams/ad.m3u
+++ b/tests/data/input/streams/ad.m3u
@ -1,5 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="ATV.ad" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",ATV (720p) [Offline]
-#EXTVLCOPT:http-referrer=http://imn.iq
-#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
-https://iptv-all.lanesh4d0w.repl.co/andorra/atv
--- a/tests/data/input/streams/ly.m3u
+++ b/tests/data/input/streams/ly.m3u
@ -1,3 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="LibyasChannel.ly",Libyas Channel
-https://master.starmena-cloud.com/hls/libyas.m3u8
--- a/tests/data/input/streams/unsorted.m3u
+++ b/tests/data/input/streams/unsorted.m3u
@ -1,3 +0,0 @@
-#EXTM3U
-#EXTINF:-1 tvg-id="",1A Network (720p)
-https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8
--- a/tests/data/input/streams_format/in.m3u
+++ b/tests/data/input/streams_format/in.m3u
@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="mn.in",Manorama News
+https://ythls.onrender.com/channel/UCP0uG-mcMImgKnJz-VjJZmQ.m3u8
--- a/tests/data/input/streams_format/nl.m3u
+++ b/tests/data/input/streams_format/nl.m3u
@ -0,0 +1,9 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (302p) [Geo-blocked]
+http://stream.tvtap.net:8081/live/nl-npo2.stream/playlist.m3u8?
+#EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (342p)
+http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8
+#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (342p) [Geo-blocked]
+http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8
+#EXTINF:-1 tvg-id="",NPO 2 (Duplicate)
+http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8
--- a/tests/data/input/streams_generate/ad.m3u
+++ b/tests/data/input/streams_generate/ad.m3u
@ -0,0 +1,5 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="Zoo.ad",Zoo (720p)
+https://iptv-all.lanesh4d0w.repl.co/andorra/zoo
+#EXTINF:-1 tvg-id="AndorraTV.ad",ATV
+https://iptv-all.lanesh4d0w.repl.co/andorra/atv
--- a/tests/data/input/streams_generate/ca.m3u
+++ b/tests/data/input/streams_generate/ca.m3u
@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="MeteoMedia.ca",Meteomedia
+http://encodercdn1.frontline.ca/encoder181/output/Meteo_Media_720p/playlist.m3u8
--- a/tests/data/input/streams_generate/in.m3u
+++ b/tests/data/input/streams_generate/in.m3u
@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="",Daawah TV
+http://51.15.246.58:8081/daawahtv/daawahtv2/playlist.m3u8
--- a/tests/data/input/streams_generate/uk.m3u
+++ b/tests/data/input/streams_generate/uk.m3u
@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD
+http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8
--- a/tests/data/input/streams_generate/unsorted.m3u
+++ b/tests/data/input/streams_generate/unsorted.m3u
@ -0,0 +1,9 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p)
+http://46.46.143.222:1935/live/mp4:ldpr.stream/blocked.m3u8
+#EXTINF:-1 tvg-id="VisitXTV.nl",Visit-X TV
+https://stream.visit-x.tv/vxtv/ngrp:live_all/30fps.m3u8
+#EXTINF:-1 tvg-id="" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",Andorra TV (720p) [Not 24/7]
+#EXTVLCOPT:http-referrer=http://imn.iq
+#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148
+http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index2.m3u8
--- a/tests/data/input/streams_report/us.m3u
+++ b/tests/data/input/streams_report/us.m3u
@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="",TUTV
+https://livestream.telvue.com/templeuni1/f7b44cfafd5c52223d5498196c8a2e7b.sdp/playlist.m3u8
--- a/tests/data/input/streams_update/br.m3u
+++ b/tests/data/input/streams_update/br.m3u
@ -0,0 +1,6 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="",VTV
+https://ythls.onrender.com/channel/UC40TUSUx490U5uR1lZt3Ajgm3u8
+#EXTINF:-1 tvg-id="",Tele2000
+#EXTVLCOPT:http-referrer=https://example2.com/
+https://servilive.com:3126/live/tele2000live.m3u8
--- a/tests/data/input/streams_update/uk.m3u
+++ b/tests/data/input/streams_update/uk.m3u
@ -0,0 +1,5 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD (720p) [Not 24/7]
+http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8
+#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD (480p) [Geo-blocked]
+http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8
--- a/tests/data/input/streams_validate/us_blocked.m3u
+++ b/tests/data/input/streams_validate/us_blocked.m3u
--- a/tests/data/input/streams_validate/wrong_id.m3u
+++ b/tests/data/input/streams_validate/wrong_id.m3u
--- a/tests/commands/api/generate.test.ts
+++ b/tests/commands/api/generate.test.ts
@ -3,14 +3,9 @@ import fs from 'fs-extra'

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
-  fs.mkdirSync('tests/__data__/output/database')
-  fs.copyFileSync(
-    'tests/__data__/input/database/api_generate.streams.db',
-    'tests/__data__/output/database/streams.db'
-  )

  const stdout = execSync(
-    'DB_DIR=tests/__data__/output/database API_DIR=tests/__data__/output/.api npm run api:generate',
+    'STREAMS_DIR=tests/__data__/input/streams_generate API_DIR=tests/__data__/output/.api npm run api:generate',
    { encoding: 'utf8' }
  )
 })
--- a/tests/commands/database/create.test.ts
+++ b/tests/commands/database/create.test.ts
@ -1,45 +0,0 @@
-import * as fs from 'fs-extra'
-import * as path from 'path'
-import { execSync } from 'child_process'
-import * as _ from 'lodash'
-
-beforeEach(() => {
-  fs.emptyDirSync('tests/__data__/output')
-  fs.mkdirSync('tests/__data__/output/database')
-
-  const stdout = execSync(
-    'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams npm run db:create',
-    { encoding: 'utf8' }
-  )
-})
-
-it('can create database', () => {
-  let output = content('tests/__data__/output/database/streams.db')
-  let expected = content('tests/__data__/expected/database/db_create.streams.db')
-
-  output = output.map(i => {
-    i._id = null
-    return i
-  })
-  expected = expected.map(i => {
-    i._id = null
-    return i
-  })
-
-  expect(_.orderBy(output, 'name')).toMatchObject(
-    expect.arrayContaining(_.orderBy(expected, 'name'))
-  )
-})
-
-function content(filepath: string) {
-  const data = fs.readFileSync(path.resolve(filepath), {
-    encoding: 'utf8'
-  })
-
-  return data
-    .split('\n')
-    .filter(l => l)
-    .map(l => {
-      return JSON.parse(l)
-    })
-}
--- a/tests/commands/playlist/format.test.ts
+++ b/tests/commands/playlist/format.test.ts
@ -0,0 +1,30 @@
+import { execSync } from 'child_process'
+import * as fs from 'fs-extra'
+import { glob } from 'glob'
+
+beforeEach(() => {
+  fs.emptyDirSync('tests/__data__/output')
+  fs.copySync('tests/__data__/input/streams_format', 'tests/__data__/output/streams')
+})
+
+it('can format playlists', () => {
+  const stdout = execSync('STREAMS_DIR=tests/__data__/output/streams npm run playlist:format', {
+    encoding: 'utf8'
+  })
+
+  const files = glob
+    .sync('tests/__data__/expected/streams_format/*.m3u')
+    .map(f => f.replace('tests/__data__/expected/streams_format/', ''))
+
+  files.forEach(filepath => {
+    expect(content(`output/streams/${filepath}`), filepath).toBe(
+      content(`expected/streams_format/${filepath}`)
+    )
+  })
+})
+
+function content(filepath: string) {
+  return fs.readFileSync(`tests/__data__/${filepath}`, {
+    encoding: 'utf8'
+  })
+}
--- a/tests/commands/playlist/generate.test.ts
+++ b/tests/commands/playlist/generate.test.ts
@ -4,13 +4,9 @@ import * as glob from 'glob'

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
-  fs.copyFileSync(
-    'tests/__data__/input/database/playlist_generate.streams.db',
-    'tests/__data__/output/streams.db'
-  )

  const stdout = execSync(
-    'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.gh-pages LOGS_DIR=tests/__data__/output/logs npm run playlist:generate',
+    'STREAMS_DIR=tests/__data__/input/streams_generate DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.gh-pages LOGS_DIR=tests/__data__/output/logs npm run playlist:generate',
    { encoding: 'utf8' }
  )
 })
--- a/tests/commands/playlist/update.test.ts
+++ b/tests/commands/playlist/update.test.ts
@ -4,15 +4,12 @@ import { glob } from 'glob'

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
-  fs.copyFileSync(
-    'tests/__data__/input/database/playlist_update.streams.db',
-    'tests/__data__/output/streams.db'
-  )
+  fs.copySync('tests/__data__/input/streams_update', 'tests/__data__/output/streams')
 })

 it('can format playlists', () => {
  const stdout = execSync(
-    'DEBUG=true DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/output/streams DB_DIR=tests/__data__/output npm run playlist:update --silent',
+    'DEBUG=true DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/output/streams npm run playlist:update --silent',
    {
      encoding: 'utf8'
    }
@ -21,11 +18,13 @@ it('can format playlists', () => {
  expect(stdout).toBe(`OUTPUT=closes #14151, closes #14110, closes #14179, closes #14178\n`)

  const files = glob
-    .sync('tests/__data__/expected/streams/*.m3u')
-    .map(f => f.replace('tests/__data__/expected/', ''))
+    .sync('tests/__data__/expected/streams_update/*.m3u')
+    .map(f => f.replace('tests/__data__/expected/streams_update/', ''))

  files.forEach(filepath => {
-    expect(content(`output/${filepath}`), filepath).toBe(content(`expected/${filepath}`))
+    expect(content(`output/streams/${filepath}`), filepath).toBe(
+      content(`expected/streams_update/${filepath}`)
+    )
  })
 })

--- a/tests/commands/playlist/validate.test.ts
+++ b/tests/commands/playlist/validate.test.ts
@ -3,7 +3,7 @@ import { execSync } from 'child_process'
 it('show an error if channel name in the blocklist', () => {
  try {
    const stdout = execSync(
-      'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams npm run playlist:validate -- tests/__data__/input/streams/us_blocked.m3u',
+      'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams_validate npm run playlist:validate -- us_blocked.m3u',
      {
        encoding: 'utf8'
      }
@ -14,7 +14,7 @@ it('show an error if channel name in the blocklist', () => {
    expect(error.status).toBe(1)
    expect(
      error.stdout.includes(
-        `loading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2     error    "Fox Sports 2 Asia (Thai)" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
+        `us_blocked.m3u\n 2     error    "Fox Sports 2 Asia (Thai)" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
      )
    ).toBe(true)
  }
@ -22,7 +22,7 @@ it('show an error if channel name in the blocklist', () => {

 it('show a warning if channel has wrong id', () => {
  const stdout = execSync(
-    'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams npm run playlist:validate -- tests/__data__/input/streams/wrong_id.m3u',
+    'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams_validate npm run playlist:validate -- wrong_id.m3u',
    {
      encoding: 'utf8'
    }
@ -30,7 +30,7 @@ it('show a warning if channel has wrong id', () => {

  expect(
    stdout.includes(
-      `loading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2     warning  "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
+      `wrong_id.m3u\n 2     warning  "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
    )
  ).toBe(true)
 })
--- a/tests/commands/report/create.test.ts
+++ b/tests/commands/report/create.test.ts
@ -1,9 +1,12 @@
 import { execSync } from 'child_process'

 it('can create report', () => {
-  const stdout = execSync('DATA_DIR=tests/__data__/input/data npm run report:create', {
-    encoding: 'utf8'
-  })
+  const stdout = execSync(
+    'DATA_DIR=tests/__data__/input/data STREAMS_DIR=tests/__data__/input/streams_report npm run report:create',
+    {
+      encoding: 'utf8'
+    }
+  )

  expect(
    stdout.includes(`
--- a/yarn.lock
+++ b/yarn.lock
@ -670,20 +670,6 @@
  resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz"
  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==

-"@seald-io/binary-search-tree@^1.0.3":
-  version "1.0.3"
-  resolved "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.3.tgz"
-  integrity sha512-qv3jnwoakeax2razYaMsGI/luWdliBLHTdC6jU55hQt1hcFqzauH/HsBollQ7IR4ySTtYhT+xyHoijpA16C+tA==
-
-"@seald-io/nedb@^4.0.2":
-  version "4.0.2"
-  resolved "https://registry.npmjs.org/@seald-io/nedb/-/nedb-4.0.2.tgz"
-  integrity sha512-gJ91fT1sgh2cLXYVcTSh7khZ8LdemI8+SojCdpZ5wy+DUQ4fSrEwGqOwbdV49NDs2BBO6GeBpSb8CnhG2IW1rw==
-  dependencies:
-    "@seald-io/binary-search-tree" "^1.0.3"
-    localforage "^1.9.0"
-    util "^0.12.4"
-
 "@sinclair/typebox@^0.27.8":
  version "0.27.8"
  resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz"
@ -947,11 +933,6 @@ async@^3.2.4:
  resolved "https://registry.npmjs.org/async/-/async-3.2.4.tgz"
  integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==

-available-typed-arrays@^1.0.5:
-  version "1.0.5"
-  resolved "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz"
-  integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==
-
 babel-jest@^29.0.0, babel-jest@^29.6.4:
  version "29.6.4"
  resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz"
@ -1074,14 +1055,6 @@ buffer-from@^1.0.0:
  resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz"
  integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==

-call-bind@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz"
-  integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==
-  dependencies:
-    function-bind "^1.1.1"
-    get-intrinsic "^1.0.2"
-
 callsites@^3.0.0:
  version "3.1.0"
  resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz"
@ -1431,13 +1404,6 @@ find-up@^4.0.0, find-up@^4.1.0:
    locate-path "^5.0.0"
    path-exists "^4.0.0"

-for-each@^0.3.3:
-  version "0.3.3"
-  resolved "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz"
-  integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==
-  dependencies:
-    is-callable "^1.1.3"
-
 foreground-child@^3.1.0:
  version "3.1.1"
  resolved "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz"
@ -1489,16 +1455,6 @@ get-caller-file@^2.0.5:
  resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz"
  integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==

-get-intrinsic@^1.0.2, get-intrinsic@^1.1.3:
-  version "1.2.1"
-  resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz"
-  integrity sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==
-  dependencies:
-    function-bind "^1.1.1"
-    has "^1.0.3"
-    has-proto "^1.0.1"
-    has-symbols "^1.0.3"
-
 get-package-type@^0.1.0:
  version "0.1.0"
  resolved "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz"
@ -1597,13 +1553,6 @@ globby@^6.1.0:
    pify "^2.0.0"
    pinkie-promise "^2.0.0"

-gopd@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz"
-  integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==
-  dependencies:
-    get-intrinsic "^1.1.3"
-
 graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.9:
  version "4.2.9"
  resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz"
@ -1619,23 +1568,6 @@ has-flag@^4.0.0:
  resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz"
  integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==

-has-proto@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz"
-  integrity sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==
-
-has-symbols@^1.0.2, has-symbols@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz"
-  integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==
-
-has-tostringtag@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz"
-  integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==
-  dependencies:
-    has-symbols "^1.0.2"
-
 has@^1.0.3:
  version "1.0.3"
  resolved "https://registry.npmjs.org/has/-/has-1.0.3.tgz"
@ -1653,11 +1585,6 @@ human-signals@^2.1.0:
  resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz"
  integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==

-immediate@~3.0.5:
-  version "3.0.6"
-  resolved "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz"
-  integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
-
 import-local@^3.0.2:
  version "3.1.0"
  resolved "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz"
@ -1679,7 +1606,7 @@ inflight@^1.0.4:
    once "^1.3.0"
    wrappy "1"

-inherits@^2.0.3, inherits@2:
+inherits@2:
  version "2.0.4"
  resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz"
  integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@ -1692,24 +1619,11 @@ iptv-playlist-parser@^0.12.3:
    is-valid-path "^0.1.1"
    validator "^13.7.0"

-is-arguments@^1.0.4:
-  version "1.1.1"
-  resolved "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz"
-  integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==
-  dependencies:
-    call-bind "^1.0.2"
-    has-tostringtag "^1.0.0"
-
 is-arrayish@^0.2.1:
  version "0.2.1"
  resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz"
  integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=

-is-callable@^1.1.3:
-  version "1.2.7"
-  resolved "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz"
-  integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==
-
 is-core-module@^2.13.0:
  version "2.13.0"
  resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz"
@ -1732,13 +1646,6 @@ is-generator-fn@^2.0.0:
  resolved "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz"
  integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==

-is-generator-function@^1.0.7:
-  version "1.0.10"
-  resolved "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz"
-  integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==
-  dependencies:
-    has-tostringtag "^1.0.0"
-
 is-glob@^2.0.0:
  version "2.0.1"
  resolved "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz"
@ -1768,13 +1675,6 @@ is-stream@^2.0.0:
  resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz"
  integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==

-is-typed-array@^1.1.3:
-  version "1.1.12"
-  resolved "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz"
-  integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==
-  dependencies:
-    which-typed-array "^1.1.11"
-
 is-valid-path@^0.1.1:
  version "0.1.1"
  resolved "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz"
@ -2272,13 +2172,6 @@ leven@^3.1.0:
  resolved "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz"
  integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==

-lie@3.1.1:
-  version "3.1.1"
-  resolved "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz"
-  integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
-  dependencies:
-    immediate "~3.0.5"
-
 lines-and-columns@^1.1.6:
  version "1.2.4"
  resolved "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz"
@ -2294,13 +2187,6 @@ load-json-file@^4.0.0:
    pify "^3.0.0"
    strip-bom "^3.0.0"

-localforage@^1.9.0:
-  version "1.10.0"
-  resolved "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz"
-  integrity sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==
-  dependencies:
-    lie "3.1.1"
-
 locate-path@^2.0.0:
  version "2.0.0"
  resolved "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz"
@ -3008,17 +2894,6 @@ universalify@^2.0.0:
  resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz"
  integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==

-util@^0.12.4:
-  version "0.12.5"
-  resolved "https://registry.npmjs.org/util/-/util-0.12.5.tgz"
-  integrity sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==
-  dependencies:
-    inherits "^2.0.3"
-    is-arguments "^1.0.4"
-    is-generator-function "^1.0.7"
-    is-typed-array "^1.1.3"
-    which-typed-array "^1.1.2"
-
 v8-compile-cache-lib@^3.0.1:
  version "3.0.1"
  resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz"
@ -3058,17 +2933,6 @@ whatwg-url@^5.0.0:
    tr46 "~0.0.3"
    webidl-conversions "^3.0.0"

-which-typed-array@^1.1.11, which-typed-array@^1.1.2:
-  version "1.1.11"
-  resolved "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz"
-  integrity sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==
-  dependencies:
-    available-typed-arrays "^1.0.5"
-    call-bind "^1.0.2"
-    for-each "^0.3.3"
-    gopd "^1.0.1"
-    has-tostringtag "^1.0.0"
-
 which@^2.0.1:
  version "2.0.2"
  resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"
				`@ -1 +0,0 @@`
				`[{"channel":"TUTV.us","url":"https://livestream.telvue.com/templeuni1/f7b44cfafd5c52223d5498196c8a2e7b.sdp/playlist.m3u8","http_referrer":null,"user_agent":null}]`