From 9fa1aab1e5c54e52fea227a3573becdbeccfde3c Mon Sep 17 00:00:00 2001 From: Dedy Martadinata S Date: Thu, 4 May 2023 14:40:09 +0700 Subject: [PATCH 1/6] Update Dockerfile, Update CI PR, use scripts to download twitchdownloader --- .github/workflows/docker-pr.yml | 4 +- Dockerfile | 63 ++++++++++++++++---------- docker-utils/fetch-twitchdownloader.sh | 39 ++++++++++++++++ 3 files changed, 81 insertions(+), 25 deletions(-) create mode 100644 docker-utils/fetch-twitchdownloader.sh diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index 4287930..71cc9a1 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -32,7 +32,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm64/v8 + platforms: linux/amd64,linux/arm/v7,linux/arm64/v8 #platforms: linux/amd64 push: false - tags: tzahi12345/youtubedl-material:nightly-pr \ No newline at end of file + tags: tzahi12345/youtubedl-material:nightly-pr diff --git a/Dockerfile b/Dockerfile index b8a53de..e15604b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,17 @@ -# Fetching our ffmpeg -FROM ubuntu:22.04 AS ffmpeg +# Fetching our utils +FROM ubuntu:20.04 AS utils ENV DEBIAN_FRONTEND=noninteractive # Use script due local build compability -COPY docker-utils/ffmpeg-fetch.sh . -RUN chmod +x ffmpeg-fetch.sh +COPY docker-utils/*.sh . +RUN chmod +x *.sh RUN sh ./ffmpeg-fetch.sh +RUN sh ./fetch-twitchdownloader.sh # Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021) # Go to 20.04 FROM ubuntu:20.04 AS base +ARG TARGETPLATFORM ARG DEBIAN_FRONTEND=noninteractive ENV UID=1000 ENV GID=1000 @@ -17,19 +19,30 @@ ENV USER=youtube ENV NO_UPDATE_NOTIFIER=true ENV PM2_HOME=/app/pm2 ENV ALLOW_CONFIG_MUTATIONS=true +# Directy fetch specific version +## https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_amd64.deb RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \ apt update && \ - apt install -y --no-install-recommends curl ca-certificates tzdata && \ - curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \ - apt install -y --no-install-recommends nodejs && \ - npm -g install npm n && \ - n 16.14.2 && \ + apt install -y --no-install-recommends curl ca-certificates tzdata libicu66 && \ apt clean && \ rm -rf /var/lib/apt/lists/* + RUN case ${TARGETPLATFORM} in \ + "linux/amd64") NODE_ARCH=amd64 ;; \ + "linux/arm") NODE_ARCH=armhf ;; \ + "linux/arm/v7") NODE_ARCH=armhf ;; \ + "linux/arm64") NODE_ARCH=arm64 ;; \ + esac \ + && curl -L https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_$NODE_ARCH.deb -o ./nodejs.deb && \ + apt update && \ + apt install -y ./nodejs.deb && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* &&\ + rm nodejs.deb; # Build frontend -FROM base as frontend +ARG BUILDPLATFORM +FROM --platform=${BUILDPLATFORM} node:16 as frontend RUN npm install -g @angular/cli WORKDIR /build COPY [ "package.json", "package-lock.json", "angular.json", "tsconfig.json", "/build/" ] @@ -49,31 +62,35 @@ RUN npm config set strict-ssl false && \ npm install --prod && \ ls -al -FROM base as python -WORKDIR /app -COPY docker-utils/GetTwitchDownloader.py . -RUN apt update && \ - apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* -RUN pip install PyGithub requests -RUN python GetTwitchDownloader.py +#FROM base as python +# armv7 need build from source +#WORKDIR /app +#COPY docker-utils/GetTwitchDownloader.py . +#RUN apt update && \ +# apt install -y --no-install-recommends python3-minimal python-is-python3 python3-pip python3-dev build-essential libffi-dev && \ +# apt clean && \ +# rm -rf /var/lib/apt/lists/* +#RUN pip install PyGithub requests +#RUN python GetTwitchDownloader.py # Final image FROM base RUN npm install -g pm2 && \ apt update && \ apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \ + pip install pycryptodomex && \ + apt remove -y --purge build-essential && \ + apt autoremove -y --purge && \ apt clean && \ rm -rf /var/lib/apt/lists/* -RUN pip install pycryptodomex WORKDIR /app # User 1000 already exist from base image -COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ] -COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ] +COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ] +COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ] +COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/TwitchDownloaderCLI", "/usr/local/bin/TwitchDownloaderCLI"] COPY --chown=$UID:$GID --from=backend ["/app/","/app/"] COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] -COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"] +#COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"] RUN chown $UID:$GID . RUN chmod +x /app/fix-scripts/*.sh # Add some persistence data diff --git a/docker-utils/fetch-twitchdownloader.sh b/docker-utils/fetch-twitchdownloader.sh new file mode 100644 index 0000000..3f25202 --- /dev/null +++ b/docker-utils/fetch-twitchdownloader.sh @@ -0,0 +1,39 @@ +!/bin/sh + +# THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out +# and also optimizing some code with this commit. +# xoxo :D + +case $(uname -m) in + x86_64) + ARCH=Linux-x64;; + aarch64) + ARCH=LinuxArm64;; + armhf) + ARCH=LinuxArm;; + armv7) + ARCH=LinuxArm;; + armv7l) + ARCH=LinuxArm;; + *) + echo "Unsupported architecture: $(uname -m)" + exit 1 +esac + +echo "(INFO) Architecture detected: $ARCH" +echo "(1/5) READY - Install unzip" +apt-get update && apt-get -y install unzip curl jq libicu66 +VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases/latest" | jq -r .tag_name) +echo "(2/5) DOWNLOAD - Acquire twitchdownloader" +curl -o twitchdownloader.zip \ + --connect-timeout 5 \ + --max-time 120 \ + --retry 5 \ + --retry-delay 0 \ + --retry-max-time 40 \ + -L "https://github.com/lay295/TwitchDownloader/releases/download/$VERSION/TwitchDownloaderCLI-$VERSION-$ARCH.zip" +unzip twitchdownloader.zip +chmod +x TwitchDownloaderCLI +echo "(3/5) Smoke test" +./TwitchDownloaderCLI --help +cp ./TwitchDownloaderCLI /usr/local/bin/TwitchDownloaderCLI From 07874d9241e9b8c070a18a3bbba130f41269899b Mon Sep 17 00:00:00 2001 From: Dedy Martadinata S Date: Sat, 20 May 2023 11:13:09 +0700 Subject: [PATCH 2/6] Revert https://github.com/Tzahi12345/YoutubeDL-Material/commit/142d708ee3ca90601254785332f9cf4b98c2e896 It become fail to set anything --- backend/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 87bb620..aabfe36 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -10,7 +10,7 @@ fi # chown current working directory to current user if [ "$*" = "$CMD" ] && [ "$(id -u)" = "0" ]; then - find . \! -user "$UID" -exec chown "$UID:$GID" '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos." + find . \! -user "$UID" -exec chown "$UID:$GID" -R '{}' + || echo "WARNING! Could not change directory ownership. If you manage permissions externally this is fine, otherwise you may experience issues when downloading or deleting videos." exec gosu "$UID:$GID" "$0" "$@" fi From 9b3816afce6f61d7805d42805b510ae9d059eef0 Mon Sep 17 00:00:00 2001 From: Dedy Martadinata S Date: Sat, 20 May 2023 11:25:16 +0700 Subject: [PATCH 3/6] Update Dockerfile --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e15604b..ed5b4af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Fetching our utils -FROM ubuntu:20.04 AS utils +FROM ubuntu:22.04 AS utils ENV DEBIAN_FRONTEND=noninteractive # Use script due local build compability COPY docker-utils/*.sh . @@ -10,7 +10,7 @@ RUN sh ./fetch-twitchdownloader.sh # Create our Ubuntu 22.04 with node 16.14.2 (that specific version is required as per: https://stackoverflow.com/a/72855258/8088021) # Go to 20.04 -FROM ubuntu:20.04 AS base +FROM ubuntu:22.04 AS base ARG TARGETPLATFORM ARG DEBIAN_FRONTEND=noninteractive ENV UID=1000 @@ -91,7 +91,6 @@ COPY --chown=$UID:$GID --from=utils [ "/usr/local/bin/TwitchDownloaderCLI", "/us COPY --chown=$UID:$GID --from=backend ["/app/","/app/"] COPY --chown=$UID:$GID --from=frontend [ "/build/backend/public/", "/app/public/" ] #COPY --chown=$UID:$GID --from=python ["/app/TwitchDownloaderCLI","/usr/local/bin/TwitchDownloaderCLI"] -RUN chown $UID:$GID . RUN chmod +x /app/fix-scripts/*.sh # Add some persistence data #VOLUME ["/app/appdata"] From fe95f04c18911cbb704c4539105aca62adf67350 Mon Sep 17 00:00:00 2001 From: Dedy Martadinata S Date: Sat, 20 May 2023 11:30:24 +0700 Subject: [PATCH 4/6] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ed5b4af..6c9b328 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV ALLOW_CONFIG_MUTATIONS=true ## https://deb.nodesource.com/node_16.x/pool/main/n/nodejs/nodejs_16.14.2-deb-1nodesource1_amd64.deb RUN groupadd -g $GID $USER && useradd --system -m -g $USER --uid $UID $USER && \ apt update && \ - apt install -y --no-install-recommends curl ca-certificates tzdata libicu66 && \ + apt install -y --no-install-recommends curl ca-certificates tzdata libicu70 && \ apt clean && \ rm -rf /var/lib/apt/lists/* RUN case ${TARGETPLATFORM} in \ From 2adbc0a02caa8d36d19260365559fc0185cdd970 Mon Sep 17 00:00:00 2001 From: Dedy Martadinata S Date: Sat, 20 May 2023 11:35:12 +0700 Subject: [PATCH 5/6] Update fetch-twitchdownloader.sh --- docker-utils/fetch-twitchdownloader.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-utils/fetch-twitchdownloader.sh b/docker-utils/fetch-twitchdownloader.sh index 3f25202..43b5a21 100644 --- a/docker-utils/fetch-twitchdownloader.sh +++ b/docker-utils/fetch-twitchdownloader.sh @@ -1,4 +1,4 @@ -!/bin/sh +!/bin/bash -eu # THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out # and also optimizing some code with this commit. @@ -22,7 +22,7 @@ esac echo "(INFO) Architecture detected: $ARCH" echo "(1/5) READY - Install unzip" -apt-get update && apt-get -y install unzip curl jq libicu66 +apt-get update && apt-get -y install unzip curl jq libicu70 VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases/latest" | jq -r .tag_name) echo "(2/5) DOWNLOAD - Acquire twitchdownloader" curl -o twitchdownloader.zip \ From e145c9c992ffc7a9adadb66e71d9985255675ae2 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 20 May 2023 19:32:00 -0600 Subject: [PATCH 6/6] Updated fetch-twitchdownloader.sh to get the latest release for the specific arch --- docker-utils/fetch-twitchdownloader.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-utils/fetch-twitchdownloader.sh b/docker-utils/fetch-twitchdownloader.sh index 43b5a21..661cdbb 100644 --- a/docker-utils/fetch-twitchdownloader.sh +++ b/docker-utils/fetch-twitchdownloader.sh @@ -1,4 +1,4 @@ -!/bin/bash -eu +#!/bin/sh # THANK YOU TALULAH (https://github.com/nottalulah) for your help in figuring this out # and also optimizing some code with this commit. @@ -23,7 +23,7 @@ esac echo "(INFO) Architecture detected: $ARCH" echo "(1/5) READY - Install unzip" apt-get update && apt-get -y install unzip curl jq libicu70 -VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases/latest" | jq -r .tag_name) +VERSION=$(curl --silent "https://api.github.com/repos/lay295/TwitchDownloader/releases" | jq -r --arg arch "$ARCH" '[.[] | select(.assets | length > 0) | select(.assets[].name | contains("CLI") and contains($arch))] | max_by(.published_at) | .tag_name') echo "(2/5) DOWNLOAD - Acquire twitchdownloader" curl -o twitchdownloader.zip \ --connect-timeout 5 \