diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index ff86dc62a..000000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: cleanup -on: - workflow_dispatch: -jobs: - cleanup: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - run: echo "::set-output name=branch_name::$(date +'bot/cleanup-%s')" - id: create-branch-name - - run: | - git config user.name 'iptv-bot[bot]' - git config user.email '84861620+iptv-bot[bot]@users.noreply.github.com' - - run: git checkout -b ${{ steps.create-branch-name.outputs.branch_name }} - - run: npm install - - run: node scripts/commands/create-database.js - - run: node scripts/commands/cleanup-database.js - - run: node scripts/commands/update-playlists.js - - run: | - git add channels/* - git commit -m "[Bot] Update playlists" - - uses: tibdex/github-app-token@v1 - if: ${{ !env.ACT }} - id: create-app-token - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - uses: repo-sync/pull-request@v2 - if: ${{ github.ref == 'refs/heads/master' }} - id: pull-request - with: - github_token: ${{ steps.create-app-token.outputs.token }} - source_branch: ${{ steps.create-branch-name.outputs.branch_name }} - destination_branch: 'master' - pr_title: '[Bot] Remove broken links' - pr_body: | - This pull request is created by [cleanup][1] workflow. - - [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} diff --git a/package.json b/package.json index d349271dc..342480489 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,6 @@ "db:matrix": "node scripts/commands/database/matrix.js", "db:update": "node scripts/commands/database/update.js", "db:export": "node scripts/commands/database/export.js", - "db:cleanup": "node scripts/commands/database/cleanup.js", "cluster:load": "node scripts/commands/cluster/load.js", "playlist:validate": "node scripts/commands/playlist/validate.js", "playlist:generate": "node scripts/commands/playlist/generate.js", diff --git a/scripts/commands/database/cleanup.js b/scripts/commands/database/cleanup.js deleted file mode 100644 index cb527a3ad..000000000 --- a/scripts/commands/database/cleanup.js +++ /dev/null @@ -1,25 +0,0 @@ -const { db, logger } = require('../../core') -const _ = require('lodash') - -async function main() { - logger.info(`loading streams...`) - await db.streams.load() - let streams = await db.streams.find({}) - - logger.info(`removing broken links...`) - let removed = 0 - const failed = _.filter(streams, { status: 'error' }) - for (const stream of failed) { - const hasDuplicate = _.find(streams, s => s.channel === stream.channel && s.status !== 'error') - if (hasDuplicate) { - await db.streams.remove({ _id: stream._id }) - removed++ - } - } - - db.streams.compact() - - logger.info(`removed ${removed} links`) -} - -main() diff --git a/tests/__data__/expected/database/db_cleanup.streams.db b/tests/__data__/expected/database/db_cleanup.streams.db deleted file mode 100644 index 9dac2dd73..000000000 --- a/tests/__data__/expected/database/db_cleanup.streams.db +++ /dev/null @@ -1,5 +0,0 @@ -{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"2ST8btby3mmsgPF0"} -{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} -{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} -{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} -{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"status":"timeout","cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/__data__/input/database/db_cleanup.streams.db b/tests/__data__/input/database/db_cleanup.streams.db deleted file mode 100644 index 492f66757..000000000 --- a/tests/__data__/input/database/db_cleanup.streams.db +++ /dev/null @@ -1,6 +0,0 @@ -{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"2ST8btby3mmsgPF0"} -{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"error","cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} -{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} -{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} -{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} -{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"status":"timeout","cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/commands/database/cleanup.test.js b/tests/commands/database/cleanup.test.js deleted file mode 100644 index e4f8948ce..000000000 --- a/tests/commands/database/cleanup.test.js +++ /dev/null @@ -1,34 +0,0 @@ -const { execSync } = require('child_process') -const fs = require('fs-extra') -const path = require('path') - -beforeEach(() => { - fs.emptyDirSync('tests/__data__/output') - fs.copyFileSync( - 'tests/__data__/input/database/db_cleanup.streams.db', - 'tests/__data__/output/streams.db' - ) - - const stdout = execSync('DB_DIR=tests/__data__/output npm run db:cleanup', { - encoding: 'utf8' - }) -}) - -it('can remove broken links from database', () => { - expect(content('tests/__data__/output/streams.db')).toEqual( - content('tests/__data__/expected/database/db_cleanup.streams.db') - ) -}) - -function content(filepath) { - const data = fs.readFileSync(path.resolve(filepath), { - encoding: 'utf8' - }) - - return data - .split('\n') - .filter(l => l) - .map(l => { - return JSON.parse(l) - }) -}