diff --git a/contrib/docker-compose/.env b/contrib/docker-compose/.env index 54f2bb10d..ee934ef7e 100644 --- a/contrib/docker-compose/.env +++ b/contrib/docker-compose/.env @@ -274,6 +274,8 @@ DB_DATABASE="pixelfed_prod" # See: https://docs.pixelfed.org/technical-documentation/config/#db_port DB_PORT="3306" +ENTRYPOINT_DEBUG=0 + ############################################################### # Mail configuration ############################################################### diff --git a/contrib/docker-compose/docker-compose.yml b/contrib/docker-compose/docker-compose.yml index e43e36c78..8df0777b3 100644 --- a/contrib/docker-compose/docker-compose.yml +++ b/contrib/docker-compose/docker-compose.yml @@ -4,10 +4,10 @@ version: "3" services: web: image: "${DOCKER_IMAGE}:${DOCKER_TAG}" - # build: - # context: ../.. - # dockerfile: contrib/docker/Dockerfile - # target: apache-runtime + build: + context: ../.. + dockerfile: contrib/docker/Dockerfile + target: apache-runtime restart: unless-stopped env_file: - "./.env" @@ -23,10 +23,10 @@ services: worker: image: "${DOCKER_IMAGE}:${DOCKER_TAG}" - # build: - # context: ../.. - # dockerfile: contrib/docker/Dockerfile - # target: apache-runtime + build: + context: ../.. + dockerfile: contrib/docker/Dockerfile + target: apache-runtime command: gosu www-data php artisan horizon restart: unless-stopped env_file: diff --git a/contrib/docker/shared/root/docker/entrypoint.d/01-permissions.sh b/contrib/docker/shared/root/docker/entrypoint.d/01-permissions.sh index 81d422ecd..25a831531 100755 --- a/contrib/docker/shared/root/docker/entrypoint.d/01-permissions.sh +++ b/contrib/docker/shared/root/docker/entrypoint.d/01-permissions.sh @@ -3,7 +3,8 @@ source /docker/helpers.sh entrypoint-set-script-name "$0" -# Ensure the two Docker volumes are owned by the runtime user +# Ensure the two Docker volumes and dot-env files are owned by the runtime user as other scripts +# will be writing to these run-as-current-user chown --verbose ${RUNTIME_UID}:${RUNTIME_GID} "./.env" run-as-current-user chown --verbose ${RUNTIME_UID}:${RUNTIME_GID} "./bootstrap/cache" run-as-current-user chown --verbose ${RUNTIME_UID}:${RUNTIME_GID} "./storage" @@ -22,5 +23,5 @@ fi for path in "${ensure_ownership_paths[@]}"; do log-info "Ensure ownership of [${path}] is correct" - run-as-current-user chown --recursive ${RUNTIME_UID}:${RUNTIME_GID} "${path}" + stream-prefix-command-output run-as-current-user chown --recursive ${RUNTIME_UID}:${RUNTIME_GID} "${path}" done diff --git a/contrib/docker/shared/root/docker/entrypoint.d/05-templating.sh b/contrib/docker/shared/root/docker/entrypoint.d/05-templating.sh index 618a2d406..cafb9d133 100755 --- a/contrib/docker/shared/root/docker/entrypoint.d/05-templating.sh +++ b/contrib/docker/shared/root/docker/entrypoint.d/05-templating.sh @@ -49,7 +49,7 @@ find "${ENTRYPOINT_TEMPLATE_DIR}" -follow -type f -print | while read -r templat cat "${template_file}" | gomplate >"${output_file_path}" # Show the diff from the envsubst command - if [[ ${ENTRYPOINT_SHOW_TEMPLATE_DIFF} = 1 ]]; then - git --no-pager diff "${template_file}" "${output_file_path}" || : + if [[ ${ENTRYPOINT_SHOW_TEMPLATE_DIFF:-1} = 1 ]]; then + git --no-pager diff --color=always "${template_file}" "${output_file_path}" || : fi done diff --git a/contrib/docker/shared/root/docker/entrypoint.d/11-first-time-setup.sh b/contrib/docker/shared/root/docker/entrypoint.d/11-first-time-setup.sh index 0e30e74ac..1a0cbb51c 100755 --- a/contrib/docker/shared/root/docker/entrypoint.d/11-first-time-setup.sh +++ b/contrib/docker/shared/root/docker/entrypoint.d/11-first-time-setup.sh @@ -3,36 +3,12 @@ source /docker/helpers.sh entrypoint-set-script-name "$0" -# if the script is running in another container, wait for it to complete -while [ -e "./storage/docker-first-time-is-running" ]; do - sleep 1 -done +await-database-ready -# We got the lock! -touch "./storage/docker-first-time-is-running" - -# Make sure to clean up on exit -trap "rm -f ./storage/docker-first-time-is-running" EXIT - -if [ ! -e "./storage/docker-storage-link-has-run" ]; then - run-as-runtime-user php artisan storage:link - touch "./storage/docker-storage-link-has-run" -fi - -if [ ! -e "./storage/docker-key-generate-has-run" ]; then - run-as-runtime-user php artisan key:generate - touch "./storage/docker-key-generate-has-run" -fi - -if [ ! -e "./storage/docker-migrate-has-run" ]; then - run-as-runtime-user php artisan migrate --force - touch "./storage/docker-migrate-has-run" -fi - -if [ ! -e "./storage/docker-import-cities-has-run" ]; then - run-as-runtime-user php artisan import:cities - touch "./storage/docker-import-cities-has-run" -fi +only-once "storage:link" run-as-runtime-user php artisan storage:link +only-once "key:generate" run-as-runtime-user php artisan key:generate +only-once "initial:migrate" run-as-runtime-user php artisan migrate --force +only-once "import:cities" run-as-runtime-user php artisan import:cities # if [ ! -e "./storage/docker-instance-actor-has-run" ]; then # run-as-runtime-user php artisan instance:actor diff --git a/contrib/docker/shared/root/docker/entrypoint.d/12-migrations.sh b/contrib/docker/shared/root/docker/entrypoint.d/12-migrations.sh new file mode 100755 index 000000000..2df379ac1 --- /dev/null +++ b/contrib/docker/shared/root/docker/entrypoint.d/12-migrations.sh @@ -0,0 +1,9 @@ +#!/bin/bash +source /docker/helpers.sh + +entrypoint-set-script-name "$0" + +await-database-ready + +declare new_migrations=0 +run-as-runtime-user php artisan migrate:status | grep No && migrations=yes || migrations=no diff --git a/contrib/docker/shared/root/docker/entrypoint.d/30-cache.sh b/contrib/docker/shared/root/docker/entrypoint.d/30-cache.sh index e933a179b..c8791e65b 100755 --- a/contrib/docker/shared/root/docker/entrypoint.d/30-cache.sh +++ b/contrib/docker/shared/root/docker/entrypoint.d/30-cache.sh @@ -3,6 +3,6 @@ source /docker/helpers.sh entrypoint-set-script-name "$0" +run-as-runtime-user php artisan config:cache run-as-runtime-user php artisan route:cache run-as-runtime-user php artisan view:cache -run-as-runtime-user php artisan config:cache diff --git a/contrib/docker/shared/root/docker/entrypoint.sh b/contrib/docker/shared/root/docker/entrypoint.sh index 173e4dbe3..0e8b1089c 100755 --- a/contrib/docker/shared/root/docker/entrypoint.sh +++ b/contrib/docker/shared/root/docker/entrypoint.sh @@ -31,6 +31,8 @@ if is-directory-empty "${ENTRYPOINT_ROOT}"; then exec "$@" fi +acquire-lock + # Start scanning for entrypoint.d files to source or run log-info "looking for shell scripts in [${ENTRYPOINT_ROOT}]" @@ -50,9 +52,9 @@ find "${ENTRYPOINT_ROOT}" -follow -type f -print | sort -V | while read -r file; log-error-and-exit "File [${file}] is not executable (please 'chmod +x' it)" fi - log-info - log-info "Sourcing [${file}]" - log-info + log-info "" + log-info "${notice_message_color}Sourcing [${file}]${color_clear}" + log-info "" source "${file}" @@ -67,9 +69,9 @@ find "${ENTRYPOINT_ROOT}" -follow -type f -print | sort -V | while read -r file; log-error-and-exit "File [${file}] is not executable (please 'chmod +x' it)" fi - log-info - log-info "Running [${file}]" - log-info + log-info "" + log-info "${notice_message_color}Executing [${file}]${color_clear}" + log-info "" "${file}" ;; @@ -80,6 +82,8 @@ find "${ENTRYPOINT_ROOT}" -follow -type f -print | sort -V | while read -r file; esac done +release-lock + log-info "Configuration complete; ready for start up" exec "$@" diff --git a/contrib/docker/shared/root/docker/helpers.sh b/contrib/docker/shared/root/docker/helpers.sh index 880027cf4..453af902d 100644 --- a/contrib/docker/shared/root/docker/helpers.sh +++ b/contrib/docker/shared/root/docker/helpers.sh @@ -6,14 +6,16 @@ set -e -o errexit -o nounset -o pipefail # Some splash of color for important messages declare -g error_message_color="\033[1;31m" declare -g warn_message_color="\033[1;34m" +declare -g notice_message_color="\033[1;34m" declare -g color_clear="\033[1;0m" # Current and previous log prefix +declare -g script_name= +declare -g script_name_previous= declare -g log_prefix= -declare -g log_prefix_previous= # dot-env files to source when reading config -declare -ra dot_env_files=( +declare -a dot_env_files=( /var/www/.env.docker /var/www/.env ) @@ -21,16 +23,24 @@ declare -ra dot_env_files=( # environment keys seen when source dot files (so we can [export] them) declare -ga seen_dot_env_variables=() +declare -g docker_state_path="$(readlink -f ./storage/docker)" +declare -g docker_locks_path="${docker_state_path}/lock" +declare -g docker_once_path="${docker_state_path}/once" + +declare -g runtime_username=$(id -un ${RUNTIME_UID}) + # @description Restore the log prefix to the previous value that was captured in [entrypoint-set-script-name ] # @arg $1 string The name (or path) of the entrypoint script being run function entrypoint-set-script-name() { - log_prefix_previous="${log_prefix}" - log_prefix="ENTRYPOINT - [$(get-entrypoint-script-name $1)] - " + script_name_previous="${script_name}" + script_name="${1}" + + log_prefix="[entrypoint / $(get-entrypoint-script-name $1)] - " } # @description Restore the log prefix to the previous value that was captured in [entrypoint-set-script-name ] function entrypoint-restore-script-name() { - log_prefix="${log_prefix_previous}" + entrypoint-set-script-name "${script_name_previous}" } # @description Run a command as the [runtime user] @@ -38,7 +48,7 @@ function entrypoint-restore-script-name() { # @exitcode 0 if the command succeeeds # @exitcode 1 if the command fails function run-as-runtime-user() { - run-command-as "$(id -un ${RUNTIME_UID})" "${@}" + run-command-as "${runtime_username}" "${@}" } # @description Run a command as the [runtime user] @@ -64,9 +74,9 @@ function run-command-as() { log-info-stderr "👷 Running [${*}] as [${target_user}]" if [[ ${target_user} != "root" ]]; then - su --preserve-environment "${target_user}" --shell /bin/bash --command "${*}" + stream-prefix-command-output su --preserve-environment "${target_user}" --shell /bin/bash --command "${*}" else - "${@}" + stream-prefix-command-output "${@}" fi exit_code=$? @@ -80,11 +90,62 @@ function run-command-as() { return $exit_code } +# @description Streams stdout from the command and echo it +# with log prefixing. +# @see stream-prefix-command-output +function stream-stdout-handler() { + local prefix="${1:-}" + + while read line; do + log-info "(stdout) ${line}" + done +} + +# @description Streams stderr from the command and echo it +# with a bit of color and log prefixing. +# @see stream-prefix-command-output +function stream-stderr-handler() { + while read line; do + log-info-stderr "(${error_message_color}stderr${color_clear}) ${line}" + done +} + +# @description Steam stdout and stderr from a command with log prefix +# and stdout/stderr prefix. If stdout or stderr is being piped/redirected +# it will automatically fall back to non-prefixed output. +# @arg $@ string The command to run +function stream-prefix-command-output() { + local stdout=stream-stdout-handler + local stderr=stream-stderr-handler + + # if stdout is being piped, print it like normal with echo + if [ ! -t 1 ]; then + stdout= echo >&1 -ne + fi + + # if stderr is being piped, print it like normal with echo + if [ ! -t 2 ]; then + stderr= echo >&2 -ne + fi + + "$@" > >($stdout) 2> >($stderr) +} + # @description Print the given error message to stderr # @arg $message string A error message. # @stderr The error message provided with log prefix function log-error() { - echo -e "${error_message_color}${log_prefix}ERROR - ${*}${color_clear}" >/dev/stderr + local msg + + if [[ $# -gt 0 ]]; then + msg="$@" + elif [[ ! -t 0 ]]; then + read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin" + else + log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty" + fi + + echo -e "${error_message_color}${log_prefix}ERROR - ${msg}${color_clear}" >/dev/stderr } # @description Print the given error message to stderr and exit 1 @@ -94,6 +155,8 @@ function log-error() { function log-error-and-exit() { log-error "$@" + show-call-stack + exit 1 } @@ -101,15 +164,35 @@ function log-error-and-exit() { # @arg $@ string A warning message. # @stderr The warning message provided with log prefix function log-warning() { - echo -e "${warn_message_color}${log_prefix}WARNING - ${*}${color_clear}" >/dev/stderr + local msg + + if [[ $# -gt 0 ]]; then + msg="$@" + elif [[ ! -t 0 ]]; then + read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin" + else + log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty" + fi + + echo -e "${warn_message_color}${log_prefix}WARNING - ${msg}${color_clear}" >/dev/stderr } # @description Print the given message to stdout unless [ENTRYPOINT_QUIET_LOGS] is set # @arg $@ string A info message. # @stdout The info message provided with log prefix unless $ENTRYPOINT_QUIET_LOGS function log-info() { + local msg + + if [[ $# -gt 0 ]]; then + msg="$@" + elif [[ ! -t 0 ]]; then + read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin" + else + log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty" + fi + if [ -z "${ENTRYPOINT_QUIET_LOGS:-}" ]; then - echo "${log_prefix}$*" + echo -e "${log_prefix}${msg}" fi } @@ -117,8 +200,18 @@ function log-info() { # @arg $@ string A info message. # @stderr The info message provided with log prefix unless $ENTRYPOINT_QUIET_LOGS function log-info-stderr() { + local msg + + if [[ $# -gt 0 ]]; then + msg="$@" + elif [[ ! -t 0 ]]; then + read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin" + else + log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty" + fi + if [ -z "${ENTRYPOINT_QUIET_LOGS:-}" ]; then - echo "${log_prefix}$*" + echo -e "${log_prefix}$msg" >/dev/stderr fi } @@ -196,3 +289,164 @@ function ensure-directory-exists() { function get-entrypoint-script-name() { echo "${1#"$ENTRYPOINT_ROOT"}" } + +# @description Ensure a command is only run once (via a 'lock' file) in the storage directory. +# The 'lock' is only written if the passed in command ($2) successfully ran. +# @arg $1 string The name of the lock file +# @arg $@ string The command to run +function only-once() { + local name="${1:-$script_name}" + local file="${docker_once_path}/${name}" + shift + + if [[ -e "${file}" ]]; then + log-info "Command [${*}] has already run once before (remove file [${file}] to run it again)" + + return 0 + fi + + ensure-directory-exists "$(dirname "${file}")" + + if ! "$@"; then + return 1 + fi + + touch "${file}" + return 0 +} + +# @description Best effort file lock to ensure *something* is not running in multiple containers. +# The script uses "trap" to clean up after itself if the script crashes +# @arg $1 string The lock identifier +function acquire-lock() { + local name="${1:-$script_name}" + local file="${docker_locks_path}/${name}" + + ensure-directory-exists "$(dirname "${file}")" + + log-info "🔑 Trying to acquire lock: ${file}: " + while [[ -e "${file}" ]]; do + log-info "🔒 Waiting on lock ${file}" + + staggered-sleep + done + + touch "${file}" + + log-info "🔐 Lock acquired [${file}]" + + on-trap "release-lock ${name}" EXIT INT QUIT TERM +} + +# @description Release a lock aquired by [acquire-lock] +# @arg $1 string The lock identifier +function release-lock() { + local name="${1:-$script_name}" + local file="${docker_locks_path}/${name}" + + log-info "🔓 Releasing lock [${file}]" + + rm -f "${file}" +} + +# @description Helper function to append multiple actions onto +# the bash [trap] logic +# @arg $1 string The command to run +# @arg $@ string The list of trap signals to register +function on-trap() { + local trap_add_cmd=$1 + shift || log-error-and-exit "${FUNCNAME} usage error" + + for trap_add_name in "$@"; do + trap -- "$( + # helper fn to get existing trap command from output + # of trap -p + extract_trap_cmd() { printf '%s\n' "${3:-}"; } + # print existing trap command with newline + eval "extract_trap_cmd $(trap -p "${trap_add_name}")" + # print the new trap command + printf '%s\n' "${trap_add_cmd}" + )" "${trap_add_name}" || + log-error-and-exit "unable to add to trap ${trap_add_name}" + done +} + +# Set the trace attribute for the above function. +# +# This is required to modify DEBUG or RETURN traps because functions don't +# inherit them unless the trace attribute is set +declare -f -t on-trap + +# @description Waits for the database to be healthy and responsive +function await-database-ready() { + log-info "❓ Waiting for database to be ready" + + case "${DB_CONNECTION:-}" in + mysql) + while ! echo "SELECT 1" | mysql --user="$DB_USERNAME" --password="$DB_PASSWORD" --host="$DB_HOST" "$DB_DATABASE" --silent >/dev/null; do + staggered-sleep + done + ;; + + pgsql) + while ! echo "SELECT 1" | psql --user="$DB_USERNAME" --password="$DB_PASSWORD" --host="$DB_HOST" "$DB_DATABASE" >/dev/null; do + staggered-sleep + done + ;; + + sqlsrv) + log-warning "Don't know how to check if SQLServer is *truely* ready or not - so will just check if we're able to connect to it" + + while ! timeout 1 bash -c "cat < /dev/null > /dev/tcp/${DB_HOST}/${DB_PORT}"; do + staggered-sleep + done + ;; + + sqlite) + log-info "sqlite are always ready" + ;; + + *) + log-error-and-exit "Unknown database type: [${DB_CONNECT}]" + ;; + esac + + log-info "✅ Successfully connected to database" +} + +# @description sleeps between 1 and 3 seconds to ensure a bit of randomness +# in multiple scripts/containers doing work almost at the same time. +function staggered-sleep() { + sleep $(get-random-number-between 1 3) +} + +# @description Helper function to get a random number between $1 and $2 +# @arg $1 int Minimum number in the range (inclusive) +# @arg $2 int Maximum number in the range (inclusive) +function get-random-number-between() { + local -i from=${1:-1} + local -i to="${2:-10}" + + shuf -i "${from}-${to}" -n 1 +} + +# @description Helper function to show the bask call stack when something +# goes wrong. Is super useful when needing to debug an issue +function show-call-stack() { + local stack_size=${#FUNCNAME[@]} + local func + local lineno + local src + + # to avoid noise we start with 1 to skip the get_stack function + for ((i = 1; i < $stack_size; i++)); do + func="${FUNCNAME[$i]}" + [ x$func = x ] && func=MAIN + + lineno="${BASH_LINENO[$((i - 1))]}" + src="${BASH_SOURCE[$i]}" + [ x"$src" = x ] && src=non_file_source + + log-error " at: ${func} ${src}:${lineno}" + done +} diff --git a/contrib/docker/shared/root/docker/install/base.sh b/contrib/docker/shared/root/docker/install/base.sh index b9b37b031..d3da207e5 100755 --- a/contrib/docker/shared/root/docker/install/base.sh +++ b/contrib/docker/shared/root/docker/install/base.sh @@ -23,6 +23,7 @@ declare -ra standardPackages=( libzip-dev locales locales-all + moreutils nano procps software-properties-common @@ -63,6 +64,8 @@ declare -ra videoProcessing=( declare -ra databaseDependencies=( libpq-dev libsqlite3-dev + mariadb-client + postgresql-client ) apt-get update