diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3aa0bbf7da4..ed8484f5b80 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,7 +9,7 @@ RUN /bin/bash --login -i -c "nvm install" # Install additional OS packages RUN apt-get update && \ export DEBIAN_FRONTEND=noninteractive && \ - apt-get -y install --no-install-recommends libicu-dev libidn11-dev ffmpeg imagemagick libvips42 libpam-dev + apt-get -y install --no-install-recommends libicu-dev libidn11-dev ffmpeg libvips42 libpam-dev # Disable download prompt for Corepack ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml index 8f05812d600..316bf831b6f 100644 --- a/.github/workflows/test-ruby.yml +++ b/.github/workflows/test-ruby.yml @@ -173,93 +173,6 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - test-imagemagick: - name: ImageMagick tests - runs-on: ubuntu-latest - - needs: - - build - - services: - postgres: - image: postgres:14-alpine - env: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres - options: >- - --health-cmd pg_isready - --health-interval 10ms - --health-timeout 3s - --health-retries 50 - ports: - - 5432:5432 - - redis: - image: redis:7-alpine - options: >- - --health-cmd "redis-cli ping" - --health-interval 10ms - --health-timeout 3s - --health-retries 50 - ports: - - 6379:6379 - - env: - DB_HOST: localhost - DB_USER: postgres - DB_PASS: postgres - COVERAGE: ${{ matrix.ruby-version == '.ruby-version' }} - RAILS_ENV: test - ALLOW_NOPAM: true - PAM_ENABLED: true - PAM_DEFAULT_SERVICE: pam_test - PAM_CONTROLLED_SERVICE: pam_test_controlled - OIDC_ENABLED: true - OIDC_SCOPE: read - SAML_ENABLED: true - CAS_ENABLED: true - BUNDLE_WITH: 'pam_authentication test' - GITHUB_RSPEC: ${{ matrix.ruby-version == '.ruby-version' && github.event.pull_request && 'true' }} - MASTODON_USE_LIBVIPS: false - - strategy: - fail-fast: false - matrix: - ruby-version: - - '3.2' - - '3.3' - - '.ruby-version' - steps: - - uses: actions/checkout@v5 - - - uses: actions/download-artifact@v6 - with: - path: './' - name: ${{ github.sha }} - - - name: Expand archived asset artifacts - run: | - tar xvzf artifacts.tar.gz - - - name: Set up Ruby environment - uses: ./.github/actions/setup-ruby - with: - ruby-version: ${{ matrix.ruby-version}} - additional-system-dependencies: ffmpeg imagemagick libpam-dev - - - name: Load database schema - run: './bin/rails db:create db:schema:load db:seed' - - - run: bin/rspec --tag attachment_processing - - - name: Upload coverage reports to Codecov - if: matrix.ruby-version == '.ruby-version' - uses: codecov/codecov-action@v5 - with: - files: coverage/lcov/mastodon.lcov - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - test-e2e: name: End to End testing runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index b9dcbe59fd2..c06bc84a339 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,8 +70,6 @@ ENV \ PATH="${PATH}:/opt/ruby/bin:/opt/mastodon/bin" \ # Optimize jemalloc 5.x performance MALLOC_CONF="narenas:2,background_thread:true,thp:never,dirty_decay_ms:1000,muzzy_decay_ms:0" \ - # Enable libvips, should not be changed - MASTODON_USE_LIBVIPS=true \ # Sidekiq will touch tmp/sidekiq_process_has_started_and_will_begin_processing_jobs to indicate it is ready. This can be used for a readiness check in Kubernetes MASTODON_SIDEKIQ_READY_FILENAME=sidekiq_process_has_started_and_will_begin_processing_jobs diff --git a/Vagrantfile b/Vagrantfile index 0a343670240..a2c0b13b146 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -29,7 +29,6 @@ sudo apt-get install \ libpq-dev \ libxml2-dev \ libxslt1-dev \ - imagemagick \ nodejs \ redis-server \ redis-tools \ diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb index e64a6b1180a..032abb75250 100644 --- a/app/lib/admin/metrics/dimension/software_versions_dimension.rb +++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb @@ -10,7 +10,7 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim protected def perform_query - [mastodon_version, ruby_version, postgresql_version, redis_version, elasticsearch_version, libvips_version, imagemagick_version, ffmpeg_version].compact + [mastodon_version, ruby_version, postgresql_version, redis_version, elasticsearch_version, libvips_version, ffmpeg_version].compact end def mastodon_version @@ -70,8 +70,6 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim end def libvips_version - return unless Rails.configuration.x.use_vips - { key: 'libvips', human_key: 'libvips', @@ -80,28 +78,6 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim } end - def imagemagick_version - return if Rails.configuration.x.use_vips - - imagemagick_binary = Paperclip.options[:is_windows] ? 'magick convert' : 'convert' - - version_output = Terrapin::CommandLine.new(imagemagick_binary, '-version').run - version_match = version_output.match(/Version: ImageMagick (\S+)/)[1].strip - - return nil unless version_match - - version = version_match - - { - key: 'imagemagick', - human_key: 'ImageMagick', - value: version, - human_value: version, - } - rescue Terrapin::CommandNotFoundError, Terrapin::ExitStatusError, Paperclip::Errors::CommandNotFoundError, Paperclip::Errors::CommandFailedError - nil - end - def ffmpeg_version version_output = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-show_program_version -v 0 -of json').run version = Oj.load(version_output, mode: :strict, symbol_keys: true).dig(:program_version, :version) diff --git a/app/models/concerns/account/avatar.rb b/app/models/concerns/account/avatar.rb index a60a289d5b0..0be46788818 100644 --- a/app/models/concerns/account/avatar.rb +++ b/app/models/concerns/account/avatar.rb @@ -4,7 +4,7 @@ module Account::Avatar extend ActiveSupport::Concern AVATAR_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze - AVATAR_LIMIT = Rails.configuration.x.use_vips ? 8.megabytes : 2.megabytes + AVATAR_LIMIT = 8.megabytes AVATAR_DIMENSIONS = [400, 400].freeze AVATAR_GEOMETRY = [AVATAR_DIMENSIONS.first, AVATAR_DIMENSIONS.last].join('x') diff --git a/app/models/concerns/account/header.rb b/app/models/concerns/account/header.rb index 662ee7caf78..066c42cb6cd 100644 --- a/app/models/concerns/account/header.rb +++ b/app/models/concerns/account/header.rb @@ -4,7 +4,7 @@ module Account::Header extend ActiveSupport::Concern HEADER_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze - HEADER_LIMIT = Rails.configuration.x.use_vips ? 8.megabytes : 2.megabytes + HEADER_LIMIT = 8.megabytes HEADER_DIMENSIONS = [1500, 500].freeze HEADER_GEOMETRY = [HEADER_DIMENSIONS.first, HEADER_DIMENSIONS.last].join('x') HEADER_MAX_PIXELS = HEADER_DIMENSIONS.first * HEADER_DIMENSIONS.last diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index 8e0e13cdb94..644be2671a9 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -39,7 +39,7 @@ class PreviewCard < ApplicationRecord include Attachmentable IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze - LIMIT = Rails.configuration.x.use_vips ? 8.megabytes : 2.megabytes + LIMIT = 8.megabytes BLURHASH_OPTIONS = { x_comp: 4, @@ -63,7 +63,7 @@ class PreviewCard < ApplicationRecord belongs_to :author_account, class_name: 'Account', optional: true has_attached_file :image, - processors: [Rails.configuration.x.use_vips ? :lazy_thumbnail : :thumbnail, :blurhash_transcoder], + processors: [:lazy_thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, validate_media_type: false diff --git a/config/application.rb b/config/application.rb index 90cfe47428f..4e58bd9f6c2 100644 --- a/config/application.rb +++ b/config/application.rb @@ -94,13 +94,7 @@ module Mastodon require 'mastodon/redis_configuration' ::REDIS_CONFIGURATION = Mastodon::RedisConfiguration.new - config.x.use_vips = ENV['MASTODON_USE_LIBVIPS'] != 'false' - - if config.x.use_vips - require_relative '../lib/paperclip/vips_lazy_thumbnail' - else - require_relative '../lib/paperclip/lazy_thumbnail' - end + require_relative '../lib/paperclip/vips_lazy_thumbnail' end config.x.cache_buster = config_for(:cache_buster) diff --git a/config/imagemagick/policy.xml b/config/imagemagick/policy.xml deleted file mode 100644 index e2aa202f274..00000000000 --- a/config/imagemagick/policy.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/config/initializers/deprecations.rb b/config/initializers/deprecations.rb index 520707e59ff..2c1057e5058 100644 --- a/config/initializers/deprecations.rb +++ b/config/initializers/deprecations.rb @@ -17,12 +17,6 @@ if ENV['REDIS_NAMESPACE'] abort message # rubocop:disable Rails/Exit end -if ENV['MASTODON_USE_LIBVIPS'] == 'false' - warn <<~MESSAGE - WARNING: Mastodon support for ImageMagick is deprecated and will be removed in future versions. Please consider using libvips instead. - MESSAGE -end - if ENV.key?('WHITELIST_MODE') warn(<<~MESSAGE.squish) WARNING: The environment variable WHITELIST_MODE has been replaced with diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index b444c5611b0..18d35a27f18 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -182,10 +182,3 @@ unless defined?(Seahorse) end end end - -# Set our ImageMagick security policy, but allow admins to override it -ENV['MAGICK_CONFIGURE_PATH'] = begin - imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR) - imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s - imagemagick_config_paths.join(File::PATH_SEPARATOR) -end diff --git a/config/initializers/vips.rb b/config/initializers/vips.rb index 09210d60ebe..cd8d2410b92 100644 --- a/config/initializers/vips.rb +++ b/config/initializers/vips.rb @@ -1,35 +1,33 @@ # frozen_string_literal: true -if Rails.configuration.x.use_vips - ENV['VIPS_BLOCK_UNTRUSTED'] = 'true' +ENV['VIPS_BLOCK_UNTRUSTED'] = 'true' - require 'vips' +require 'vips' - unless Vips.at_least_libvips?(8, 13) - abort <<~ERROR.squish # rubocop:disable Rails/Exit - Incompatible libvips version (#{Vips.version_string}), please install libvips >= 8.13 - ERROR - end +unless Vips.at_least_libvips?(8, 13) + abort <<~ERROR.squish # rubocop:disable Rails/Exit + Incompatible libvips version (#{Vips.version_string}), please install libvips >= 8.13 + ERROR +end - Vips.block('VipsForeign', true) +Vips.block('VipsForeign', true) - %w( - VipsForeignLoadNsgif - VipsForeignLoadJpeg - VipsForeignLoadPng - VipsForeignLoadWebp - VipsForeignLoadHeif - VipsForeignSavePng - VipsForeignSaveSpng - VipsForeignSaveJpeg - VipsForeignSaveWebp - ).each do |operation| - Vips.block(operation, false) - end - - Vips.block_untrusted(true) +%w( + VipsForeignLoadNsgif + VipsForeignLoadJpeg + VipsForeignLoadPng + VipsForeignLoadWebp + VipsForeignLoadHeif + VipsForeignSavePng + VipsForeignSaveSpng + VipsForeignSaveJpeg + VipsForeignSaveWebp +).each do |operation| + Vips.block(operation, false) end +Vips.block_untrusted(true) + # In some places of the code, we rescue this exception, but we don't always # load libvips, so it may be an undefined constant: unless defined?(Vips) diff --git a/lib/paperclip/blurhash_transcoder.rb b/lib/paperclip/blurhash_transcoder.rb index b4ff4a12a0e..e9ff1dd9dd8 100644 --- a/lib/paperclip/blurhash_transcoder.rb +++ b/lib/paperclip/blurhash_transcoder.rb @@ -19,14 +19,8 @@ module Paperclip private def blurhash_params - if Rails.configuration.x.use_vips - image = Vips::Image.thumbnail(@file.path, 100) - [image.width, image.height, image.colourspace(:srgb).extract_band(0, n: 3).to_a.flatten] - else - pixels = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*') - geometry = options.fetch(:file_geometry_parser).from_file(@file) - [geometry.width, geometry.height, pixels] - end + image = Vips::Image.thumbnail(@file.path, 100) + [image.width, image.height, image.colourspace(:srgb).extract_band(0, n: 3).to_a.flatten] end end end diff --git a/lib/paperclip/color_extractor.rb b/lib/paperclip/color_extractor.rb index 62daa077952..1c9ef4bd3d6 100644 --- a/lib/paperclip/color_extractor.rb +++ b/lib/paperclip/color_extractor.rb @@ -10,7 +10,7 @@ module Paperclip BINS = 10 def make - background_palette, foreground_palette = Rails.configuration.x.use_vips ? palettes_from_libvips : palettes_from_imagemagick + background_palette, foreground_palette = palettes_from_libvips background_color = background_palette.first || foreground_palette.first foreground_colors = [] @@ -93,17 +93,6 @@ module Paperclip [background_palette, foreground_palette] end - def palettes_from_imagemagick - depth = 8 - - # Determine background palette by getting colors close to the image's edge only - background_palette = palette_from_im_histogram(convert(':source -alpha set -gravity Center -region 75%x75% -fill None -colorize 100% -alpha transparent +region -format %c -colors :quantity -depth :depth histogram:info:', source: File.expand_path(@file.path), quantity: 10, depth: depth), 10) - - # Determine foreground palette from the whole image - foreground_palette = palette_from_im_histogram(convert(':source -format %c -colors :quantity -depth :depth histogram:info:', source: File.expand_path(@file.path), quantity: 10, depth: depth), 10) - [background_palette, foreground_palette] - end - def downscaled_image image = Vips::Image.new_from_file(@file.path, access: :random).thumbnail_image(100) diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb index a712cdde1dc..cbabb1d0dd3 100644 --- a/spec/models/media_attachment_spec.rb +++ b/spec/models/media_attachment_spec.rb @@ -219,9 +219,7 @@ RSpec.describe MediaAttachment, :attachment_processing do describe 'ogg with cover art' do let(:media) { Fabricate(:media_attachment, file: attachment_fixture('boop.ogg')) } let(:expected_media_duration) { 0.235102 } - - # The libvips and ImageMagick implementations produce different results - let(:expected_background_color) { Rails.configuration.x.use_vips ? '#268cd9' : '#3088d4' } + let(:expected_background_color) { '#268cd9' } it 'sets correct file metadata' do expect(media) diff --git a/spec/models/preview_card_spec.rb b/spec/models/preview_card_spec.rb index 0fe76c37b07..bac29046eac 100644 --- a/spec/models/preview_card_spec.rb +++ b/spec/models/preview_card_spec.rb @@ -3,12 +3,6 @@ require 'rails_helper' RSpec.describe PreviewCard do - describe 'file size limit', :attachment_processing do - it 'is set differently whether vips is enabled or not' do - expect(described_class::LIMIT).to eq(Rails.configuration.x.use_vips ? 8.megabytes : 2.megabytes) - end - end - describe 'Validations' do describe 'url' do it { is_expected.to allow_values('http://example.host/path', 'https://example.host/path').for(:url) } diff --git a/spec/requests/api/v1/media_spec.rb b/spec/requests/api/v1/media_spec.rb index 347ff4b2797..89e34998db7 100644 --- a/spec/requests/api/v1/media_spec.rb +++ b/spec/requests/api/v1/media_spec.rb @@ -102,7 +102,7 @@ RSpec.describe 'Media' do allow(user.account).to receive(:media_attachments).and_return(media_attachments) end - context 'when imagemagick cannot identify the file type' do + context 'when file type cannot be identified' do it 'returns http unprocessable entity' do allow(media_attachments).to receive(:create!).and_raise(Paperclip::Errors::NotIdentifiedByImageMagickError) diff --git a/spec/requests/api/v2/media_spec.rb b/spec/requests/api/v2/media_spec.rb index 04e48bc02c3..1f149b9634d 100644 --- a/spec/requests/api/v2/media_spec.rb +++ b/spec/requests/api/v2/media_spec.rb @@ -71,7 +71,7 @@ RSpec.describe 'Media API', :attachment_processing do allow(user.account).to receive(:media_attachments).and_return(media_attachments) end - context 'when imagemagick cannot identify the file type' do + context 'when file type cannot be identified' do before do allow(media_attachments).to receive(:create!).and_raise(Paperclip::Errors::NotIdentifiedByImageMagickError) end