diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a2c48a1..425c09850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,54 @@ All notable changes to this project will be documented in this file. +## [4.1.3] - 2023-07-06 + +### Added + +- Add fallback redirection when getting a webfinger query `LOCAL_DOMAIN@LOCAL_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23600)) + +### Changed + +- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058)) +- Change AccessTokensVacuum to also delete expired tokens ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24868)) +- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852)) +- Change automatic post deletion thresholds and load detection ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24614)) +- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510)) +- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216)) + +### Removed + +- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070)) + +### Fixed + +- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464)) +- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519)) +- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477)) +- Fix multiple inefficiencies in automatic post cleanup worker ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24785), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24840)) +- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361)) +- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273)) +- Fix `tootctl accounts approve --number N` not aproving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605)) +- Fix reports not being closed when performing batch suspensions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24988)) +- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015)) +- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016)) +- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060)) +- Fix “Authorized applications” crashing when listing apps with certain admin API scopes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25713)) +- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499)) +- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431)) +- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637)) +- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342)) + +### Security + +- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463)) +- Update dependencies +- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756)) +- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462) +- Fix timeout handling of outbound HTTP requests (CVE-2023-36461) +- Fix arbitrary file creation through media processing (CVE-2023-36460) +- Fix possible XSS in preview cards (CVE-2023-36459) + ## [4.1.2] - 2023-04-04 ### Fixed diff --git a/app/helpers/formatting_helper.rb b/app/helpers/formatting_helper.rb index 5b2ac1a2a..f44cf7973 100644 --- a/app/helpers/formatting_helper.rb +++ b/app/helpers/formatting_helper.rb @@ -54,6 +54,10 @@ module FormattingHelper end def account_field_value_format(field, with_rel_me: true) - html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false) + if field.verified? && !field.account.local? + TextFormatter.shortened_link(field.value_for_verification) + else + html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false) + end end end diff --git a/app/lib/request.rb b/app/lib/request.rb index 4bde6fc91..425effa1a 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -7,11 +7,48 @@ require 'resolv' # Monkey-patch the HTTP.rb timeout class to avoid using a timeout block # around the Socket#open method, since we use our own timeout blocks inside # that method +# +# Also changes how the read timeout behaves so that it is cumulative (closer +# to HTTP::Timeout::Global, but still having distinct timeouts for other +# operation types) class HTTP::Timeout::PerOperation def connect(socket_class, host, port, nodelay = false) @socket = socket_class.open(host, port) @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay end + + # Reset deadline when the connection is re-used for different requests + def reset_counter + @deadline = nil + end + + # Read data from the socket + def readpartial(size, buffer = nil) + @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout + + timeout = false + loop do + result = @socket.read_nonblock(size, buffer, exception: false) + + return :eof if result.nil? + + remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC) + raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0 + return result if result != :wait_readable + + # marking the socket for timeout. Why is this not being raised immediately? + # it seems there is some race-condition on the network level between calling + # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting + # for reads, and when waiting for x seconds, it returns nil suddenly without completing + # the x seconds. In a normal case this would be a timeout on wait/read, but it can + # also mean that the socket has been closed by the server. Therefore we "mark" the + # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no + # timeout. Else, the first timeout was a proper timeout. + # This hack has to be done because io/wait#wait_readable doesn't provide a value for when + # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks. + timeout = true unless @socket.to_io.wait_readable(remaining_time) + end + end end class Request diff --git a/app/lib/text_formatter.rb b/app/lib/text_formatter.rb index 0404cbace..3570632dd 100644 --- a/app/lib/text_formatter.rb +++ b/app/lib/text_formatter.rb @@ -48,6 +48,26 @@ class TextFormatter html.html_safe # rubocop:disable Rails/OutputSafety end + class << self + include ERB::Util + + def shortened_link(url, rel_me: false) + url = Addressable::URI.parse(url).to_s + rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL + + prefix = url.match(URL_PREFIX_REGEX).to_s + display_url = url[prefix.length, 30] + suffix = url[prefix.length + 30..-1] + cutoff = url[prefix.length..-1].length > 30 + + <<~HTML.squish + #{h(display_url)} + HTML + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError + h(url) + end + end + private def rewrite @@ -70,19 +90,7 @@ class TextFormatter end def link_to_url(entity) - url = Addressable::URI.parse(entity[:url]).to_s - rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL - - prefix = url.match(URL_PREFIX_REGEX).to_s - display_url = url[prefix.length, 30] - suffix = url[prefix.length + 30..-1] - cutoff = url[prefix.length..-1].length > 30 - - <<~HTML.squish - #{h(display_url)} - HTML - rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError - h(entity[:url]) + TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?) end def link_to_hashtag(entity) diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index 9cafedc20..f93ee4c91 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -22,15 +22,14 @@ module Attachmentable included do def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName - options = { validate_media_type: false }.merge(options) super(name, options) - send(:"before_#{name}_post_process") do + + send(:"before_#{name}_validate") do attachment = send(name) check_image_dimension(attachment) set_file_content_type(attachment) obfuscate_file_name(attachment) set_file_extension(attachment) - Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self) end end end diff --git a/app/serializers/rest/preview_card_serializer.rb b/app/serializers/rest/preview_card_serializer.rb index 8413b23d8..08bc07edd 100644 --- a/app/serializers/rest/preview_card_serializer.rb +++ b/app/serializers/rest/preview_card_serializer.rb @@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer def image object.image? ? full_asset_url(object.image.url(:original)) : nil end + + def html + Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED) + end end diff --git a/config/application.rb b/config/application.rb index d3c99baa1..8c4ec27e7 100644 --- a/config/application.rb +++ b/config/application.rb @@ -28,6 +28,7 @@ require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/gif_transcoder' +require_relative '../lib/paperclip/media_type_spoof_detector_extensions' require_relative '../lib/paperclip/transcoder' require_relative '../lib/paperclip/type_corrector' require_relative '../lib/paperclip/response_with_limit_adapter' diff --git a/config/imagemagick/policy.xml b/config/imagemagick/policy.xml new file mode 100644 index 000000000..1052476b3 --- /dev/null +++ b/config/imagemagick/policy.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 093d2ba9a..f2da410db 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -153,3 +153,10 @@ unless defined?(Seahorse) end end end + +# Set our ImageMagick security policy, but allow admins to override it +ENV['MAGICK_CONFIGURE_PATH'] = begin + imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR) + imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s + imagemagick_config_paths.join(File::PATH_SEPARATOR) +end diff --git a/dist/nginx.conf b/dist/nginx.conf index bed4bd3db..fc68e9a6d 100644 --- a/dist/nginx.conf +++ b/dist/nginx.conf @@ -109,6 +109,8 @@ server { location ~ ^/system/ { add_header Cache-Control "public, max-age=2419200, immutable"; add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; + add_header X-Content-Type-Options nosniff; + add_header Content-Security-Policy "default-src 'none'; form-action 'none'"; try_files $uri =404; } diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index f71762d99..3b1338e5b 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 2 + 3 end def flags diff --git a/lib/paperclip/media_type_spoof_detector_extensions.rb b/lib/paperclip/media_type_spoof_detector_extensions.rb new file mode 100644 index 000000000..a406ef312 --- /dev/null +++ b/lib/paperclip/media_type_spoof_detector_extensions.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Paperclip + module MediaTypeSpoofDetectorExtensions + def calculated_content_type + return @calculated_content_type if defined?(@calculated_content_type) + + @calculated_content_type = type_from_file_command.chomp + + # The `file` command fails to recognize some MP3 files as such + @calculated_content_type = type_from_marcel if @calculated_content_type == 'application/octet-stream' && type_from_marcel == 'audio/mpeg' + @calculated_content_type + end + + def type_from_marcel + @type_from_marcel ||= Marcel::MimeType.for Pathname.new(@file.path), + name: @file.path + end + end +end + +Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions) diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index b3b55f82f..f4768aa60 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -19,10 +19,7 @@ module Paperclip def make metadata = VideoMetadataExtractor.new(@file.path) - unless metadata.valid? - Paperclip.log("Unsupported file #{@file.path}") - return File.open(@file.path) - end + raise Paperclip::Error, "Error while transcoding #{@file.path}: unsupported file" unless metadata.valid? update_attachment_type(metadata) update_options_from_metadata(metadata) diff --git a/lib/public_file_server_middleware.rb b/lib/public_file_server_middleware.rb index 3799230a2..7e02e37a0 100644 --- a/lib/public_file_server_middleware.rb +++ b/lib/public_file_server_middleware.rb @@ -32,6 +32,11 @@ class PublicFileServerMiddleware end end + # Override the default CSP header set by the CSP middleware + headers['Content-Security-Policy'] = "default-src 'none'; form-action 'none'" if request_path.start_with?(paperclip_root_url) + + headers['X-Content-Type-Options'] = 'nosniff' + [status, headers, response] end diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb index 762f69cfc..53508d3e4 100644 --- a/lib/sanitize_ext/sanitize_config.rb +++ b/lib/sanitize_ext/sanitize_config.rb @@ -106,26 +106,26 @@ class Sanitize ] ) - MASTODON_OEMBED ||= freeze_config merge( - RELAXED, - elements: RELAXED[:elements] + %w(audio embed iframe source video), + MASTODON_OEMBED ||= freeze_config( + elements: %w(audio embed iframe source video), - attributes: merge( - RELAXED[:attributes], + attributes: { 'audio' => %w(controls), 'embed' => %w(height src type width), 'iframe' => %w(allowfullscreen frameborder height scrolling src width), 'source' => %w(src type), 'video' => %w(controls height loop width), - 'div' => [:data] - ), + }, - protocols: merge( - RELAXED[:protocols], + protocols: { 'embed' => { 'src' => HTTP_PROTOCOLS }, 'iframe' => { 'src' => HTTP_PROTOCOLS }, - 'source' => { 'src' => HTTP_PROTOCOLS } - ) + 'source' => { 'src' => HTTP_PROTOCOLS }, + }, + + add_attributes: { + 'iframe' => { 'sandbox' => 'allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox allow-forms' }, + } ) LINK_REL_TRANSFORMER = lambda do |env| diff --git a/spec/fixtures/files/boop.mp3 b/spec/fixtures/files/boop.mp3 new file mode 100644 index 000000000..ba106a3a3 Binary files /dev/null and b/spec/fixtures/files/boop.mp3 differ diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb index 2dfc6cf92..90e4f2f47 100644 --- a/spec/models/media_attachment_spec.rb +++ b/spec/models/media_attachment_spec.rb @@ -152,6 +152,26 @@ RSpec.describe MediaAttachment, paperclip_processing: true do end end + describe 'mp3 with large cover art' do + let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.mp3')) } + + it 'detects it as an audio file' do + expect(media.type).to eq 'audio' + end + + it 'sets meta for the duration' do + expect(media.file.meta['original']['duration']).to be_within(0.05).of(0.235102) + end + + it 'extracts thumbnail' do + expect(media.thumbnail.present?).to be true + end + + it 'gives the file a random name' do + expect(media.file_file_name).to_not eq 'boop.mp3' + end + end + describe 'jpeg' do let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) }