Merge pull request #872 from Tzahi12345/twitch-downloader-fix

Twitch chat download fixes

Dockerfile

@@ -2,7 +2,7 @@
 FROM ubuntu:22.04 AS ffmpeg
 ENV DEBIAN_FRONTEND=noninteractive
 # Use script due local build compability
-COPY ffmpeg-fetch.sh .
+COPY docker-utils/ffmpeg-fetch.sh .
 RUN chmod +x ffmpeg-fetch.sh
 RUN sh ./ffmpeg-fetch.sh
 
@@ -47,6 +47,15 @@ RUN npm config set strict-ssl false && \
     npm install --prod && \
     ls -al
 
+FROM base as python
+WORKDIR /app
+COPY docker-utils/GetTwitchDownloader.py .
+RUN apt update && \
+    apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install PyGithub requests
+RUN python GetTwitchDownloader.py
 
 # Final image
 FROM base
@@ -55,7 +64,7 @@ RUN npm install -g pm2 && \
     apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
-RUN pip install tdh-tcd pycryptodomex
+RUN pip install pycryptodomex
 WORKDIR /app
 # User 1000 already exist from base image
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]

backend/config.js

@@ -208,9 +208,6 @@ const DEFAULT_CONFIG = {
         "API_key": "",
         "use_youtube_API": false,
         "youtube_API_key": "",
-        "use_twitch_API": false,
-        "twitch_client_ID": "",
-        "twitch_client_secret": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
         "generate_NFO_files": false,

backend/consts.js

@@ -110,18 +110,6 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
-    'ytdl_use_twitch_api': {
-        'key': 'ytdl_use_twitch_api',
-        'path': 'YoutubeDLMaterial.API.use_twitch_API'
-    },
-    'ytdl_twitch_client_id': {
-        'key': 'ytdl_twitch_client_id',
-        'path': 'YoutubeDLMaterial.API.twitch_client_ID'
-    },
-    'ytdl_twitch_client_secret': {
-        'key': 'ytdl_twitch_client_secret',
-        'path': 'YoutubeDLMaterial.API.twitch_client_secret'
-    },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',
         'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'

backend/downloader.js

@@ -350,7 +350,7 @@ async function downloadQueuedFile(download_uid) {
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
                     if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
-                        && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                        && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                             let vodId = url.split('twitch.tv/videos/')[1];
                             vodId = vodId.split('?')[0];
                             twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);

backend/test/tests.js

@@ -508,7 +508,7 @@ describe('Downloader', function() {
     });
     describe('Twitch', async function () {
         const twitch_api = require('../twitch');
-        const example_vod = '1493770675';
+        const example_vod = '1710641401';
         it('Download VOD', async function() {
             const sample_path = path.join('test', 'sample.twitch_chat.json');
             if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);

backend/twitch.js

@@ -4,19 +4,28 @@ const logger = require('./logger');
 const moment = require('moment');
 const fs = require('fs-extra')
 const path = require('path');
+const { promisify } = require('util');
+const child_process = require('child_process');
 
-async function getCommentsForVOD(clientID, clientSecret, vodId) {
-    const { promisify } = require('util');
-    const child_process = require('child_process');
+async function getCommentsForVOD(vodId) {
     const exec = promisify(child_process.exec);
     
     // Reject invalid params to prevent command injection attack
-    if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) {
-        logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!');
+    if (!vodId.match(/^[0-9a-z]+$/)) {
+        logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!');
         return null;
     }
 
-    const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
+    const is_windows = process.platform === 'win32';
+    const cliExt = is_windows ? '.exe' : ''
+    const cliPath = `TwitchDownloaderCLI${cliExt}`
+
+    if (!fs.existsSync(cliPath)) {
+        logger.error(`${cliPath} does not exist. Twitch chat download failed! Get it here: https://github.com/lay295/TwitchDownloader`);
+        return null;
+    }
+
+    const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]});
 
     if (result['stderr']) {
         logger.error(`Failed to download twitch comments for ${vodId}`);
@@ -73,9 +82,7 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
 async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
     const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
     const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
-    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
-    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
-    const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
+    const chat = await getCommentsForVOD(vodId);
 
     // save file if needed params are included
     let file_path = null;

docker-utils/GetTwitchDownloader.py

@@ -0,0 +1,53 @@
+import platform
+import requests
+import shutil
+import os
+import re
+
+from github import Github
+
+machine = platform.machine()
+
+def isARM():
+    return True if machine.startswith('arm') else False
+
+def getLatestFileInRepo(repo, search_string):
+    # Create an unauthenticated instance of the Github object
+    g = Github(os.environ.get('GH_TOKEN'))
+
+    # Replace with the repository owner and name
+    repo = g.get_repo(repo)
+
+    # Get all releases of the repository
+    releases = repo.get_releases()
+
+    # Loop through the releases in reverse order (from latest to oldest)
+    for release in list(releases):
+        # Get the release assets (files attached to the release)
+        assets = release.get_assets()
+
+        # Loop through the assets
+        for asset in assets:
+            if re.search(search_string, asset.name):
+                print(f'Downloading: {asset.name}')
+                response = requests.get(asset.browser_download_url)
+                with open(asset.name, 'wb') as f:
+                    f.write(response.content)
+                print(f'Download complete: {asset.name}. Unzipping...')
+                shutil.unpack_archive(asset.name, './')
+                print(f'Unzipping complete!')
+                os.remove(asset.name)
+                break
+        else:
+            continue
+        break
+    else:
+        # If no matching release is found, print a message
+        print(f'No release found with {search_string}')
+
+def getLatestCLIRelease():
+    isArm = isARM()
+    searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip"
+    getLatestFileInRepo("lay295/TwitchDownloader", searchString)
+
+getLatestCLIRelease()

src/app/player/player.component.html

@@ -38,7 +38,7 @@
                                     </ng-container>
                                     <ng-container *ngIf="db_file || playlist[currentIndex]"></ng-container>
                                     <button (click)="openFileInfoDialog()" *ngIf="db_file || db_playlist" mat-icon-button><mat-icon>info</mat-icon></button>
-                                    <button *ngIf="db_file && db_file.url.includes('twitch.tv') && postsService['config']['API']['use_twitch_API']" (click)="drawer.toggle()" mat-icon-button><mat-icon>chat</mat-icon></button>
+                                    <button *ngIf="db_file && db_file.url.includes('twitch.tv')" (click)="drawer.toggle()" mat-icon-button><mat-icon>chat</mat-icon></button>
                                 </div>
                             </div>
                         </div>
@@ -51,7 +51,7 @@
 
                     <app-concurrent-stream *ngIf="db_file && api && postsService.config" (setPlaybackRate)="setPlaybackRate($event)" (togglePlayback)="togglePlayback($event)" (setPlaybackTimestamp)="setPlaybackTimestamp($event)" [playing]="api.state === 'playing'" [uid]="uid" [playback_timestamp]="api.time.current/1000" [server_mode]="!postsService.config.Advanced.multi_user_mode || postsService.isLoggedIn"></app-concurrent-stream>
 
-                    <mat-drawer #drawer class="example-sidenav" mode="side" position="end" [opened]="db_file && db_file['chat_exists'] && postsService['config']['API']['use_twitch_API']">
+                    <mat-drawer #drawer class="example-sidenav" mode="side" position="end" [opened]="db_file && db_file['chat_exists']">
                         <ng-container *ngIf="api_ready && db_file && db_file.url.includes('twitch.tv')">
                             <app-twitch-chat #twitchchat [db_file]="db_file" [current_timestamp]="api.currentTime" [sub]="subscription"></app-twitch-chat>
                         </ng-container>

src/app/settings/settings.component.html

@@ -269,25 +269,9 @@
                             <mat-hint><a target="_blank" href="https://developers.google.com/youtube/v3/getting-started"><ng-container i18n="Youtube API Key setting hint">Generating a key is easy!</ng-container></a></mat-hint>
                         </mat-form-field>
                     </div>
-                    <div class="col-12 mt-3">
-                        <mat-checkbox color="accent" [(ngModel)]="new_config['API']['use_twitch_API']"><ng-container i18n="Use Twitch API setting">Use Twitch API</ng-container></mat-checkbox>
-                    </div>
-                    <div *ngIf="new_config['API']['use_twitch_API']" class="col-12 mt-1">
+                    <div class="col-12 mt-1">
                         <mat-checkbox color="accent" [(ngModel)]="new_config['API']['twitch_auto_download_chat']"><ng-container i18n="Auto download Twitch Chat setting">Auto-download Twitch Chat</ng-container></mat-checkbox>
                     </div>
-                    <div class="col-12">
-                        <mat-form-field class="text-field" color="accent">
-                            <mat-label i18n="Twitch Client ID">Twitch Client ID</mat-label>
-                            <input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_ID']" matInput required>
-                            <mat-hint><a target="_blank" href="https://dev.twitch.tv/docs/api/"><ng-container i18n="Twitch Client ID setting hint">Generating an ID/secret is easy!</ng-container></a></mat-hint>
-                        </mat-form-field>
-                    </div>
-                    <div class="col-12 mt-2">
-                        <mat-form-field class="text-field" color="accent">
-                            <mat-label i18n="Twitch Client Secret">Twitch Client Secret</mat-label>
-                            <input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_secret']" matInput required>
-                        </mat-form-field>
-                    </div>
                     <div class="col-12 mt-2">
                         <mat-checkbox color="accent" [(ngModel)]="new_config['API']['use_sponsorblock_API']" matTooltip="Enables a button to skip ads when viewing supported videos." i18n-matTooltip="SponsorBlock API tooltip"><ng-container i18n="Use SponsorBlock API setting">Use SponsorBlock API</ng-container></mat-checkbox>
                     </div>