From 30c191aaa03beb8bc4de4bf33db671fb98bafc96 Mon Sep 17 00:00:00 2001 From: jsgoldstein Date: Thu, 24 Aug 2023 10:40:04 -0400 Subject: [PATCH 01/10] Add new public status index (#26344) Co-authored-by: Eugen Rochko Co-authored-by: Claire --- app/chewy/accounts_index.rb | 2 +- app/chewy/public_statuses_index.rb | 50 +++++++++++++ app/chewy/statuses_index.rb | 22 +++--- .../api/v1/accounts/credentials_controller.rb | 1 + .../settings/privacy_controller.rb | 2 +- .../public_statuses_index_importer.rb | 41 ++++++++++ app/lib/search_query_transformer.rb | 49 +++++++++--- app/lib/vacuum/statuses_vacuum.rb | 9 ++- app/models/account.rb | 1 + .../concerns/account_statuses_search.rb | 44 +++++++++++ app/models/concerns/status_search_concern.rb | 54 +++++++++++++ app/models/status.rb | 37 ++------- .../activitypub/actor_serializer.rb | 8 +- app/services/batched_remove_status_service.rb | 5 +- app/services/search_service.rb | 32 +++----- app/services/statuses_search_service.rb | 75 +++++++++++++++++++ app/views/settings/privacy/show.html.haml | 3 + .../add_to_public_statuses_index_worker.rb | 15 ++++ ...emove_from_public_statuses_index_worker.rb | 15 ++++ app/workers/scheduler/indexing_scheduler.rb | 2 +- config/locales/simple_form.en.yml | 2 + lib/mastodon/cli/search.rb | 1 + spec/chewy/public_statuses_index_spec.rb | 31 ++++++++ .../public_statuses_index_importer_spec.rb | 16 ++++ spec/lib/search_query_transformer_spec.rb | 4 +- .../concerns/account_statuses_search_spec.rb | 66 ++++++++++++++++ ...dd_to_public_statuses_index_worker_spec.rb | 42 +++++++++++ ..._from_public_statuses_index_worker_spec.rb | 42 +++++++++++ 28 files changed, 584 insertions(+), 87 deletions(-) create mode 100644 app/chewy/public_statuses_index.rb create mode 100644 app/lib/importer/public_statuses_index_importer.rb create mode 100644 app/models/concerns/account_statuses_search.rb create mode 100644 app/models/concerns/status_search_concern.rb create mode 100644 app/services/statuses_search_service.rb create mode 100644 app/workers/add_to_public_statuses_index_worker.rb create mode 100644 app/workers/remove_from_public_statuses_index_worker.rb create mode 100644 spec/chewy/public_statuses_index_spec.rb create mode 100644 spec/lib/importer/public_statuses_index_importer_spec.rb create mode 100644 spec/models/concerns/account_statuses_search_spec.rb create mode 100644 spec/workers/add_to_public_statuses_index_worker_spec.rb create mode 100644 spec/workers/remove_from_public_statuses_index_worker_spec.rb diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index 1f8571c09..61e3399aa 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -62,6 +62,6 @@ class AccountsIndex < Chewy::Index field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }) field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' } field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' } - field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' } + field(:text, type: 'text', analyzer: 'whitespace', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' } end end diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb new file mode 100644 index 000000000..1fad5de3a --- /dev/null +++ b/app/chewy/public_statuses_index.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class PublicStatusesIndex < Chewy::Index + settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: { + filter: { + english_stop: { + type: 'stop', + stopwords: '_english_', + }, + + english_stemmer: { + type: 'stemmer', + language: 'english', + }, + + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english', + }, + }, + + analyzer: { + content: { + tokenizer: 'uax_url_email', + filter: %w( + english_possessive_stemmer + lowercase + asciifolding + cjk_width + english_stop + english_stemmer + ), + }, + }, + } + + index_scope ::Status.unscoped + .kept + .indexable + .includes(:media_attachments, :preloadable_poll, :preview_cards) + + root date_detection: false do + field(:id, type: 'keyword') + field(:account_id, type: 'long') + field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') } + field(:language, type: 'keyword') + field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties }) + field(:created_at, type: 'date') + end +end diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 9f680efa5..130f8801d 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -1,23 +1,24 @@ # frozen_string_literal: true class StatusesIndex < Chewy::Index - include FormattingHelper - settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: { filter: { english_stop: { type: 'stop', stopwords: '_english_', }, + english_stemmer: { type: 'stemmer', language: 'english', }, + english_possessive_stemmer: { type: 'stemmer', language: 'possessive_english', }, }, + analyzer: { content: { tokenizer: 'uax_url_email', @@ -35,7 +36,7 @@ class StatusesIndex < Chewy::Index # We do not use delete_if option here because it would call a method that we # expect to be called with crutches without crutches, causing n+1 queries - index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll) + index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll, :preview_cards) crutch :mentions do |collection| data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id) @@ -63,13 +64,12 @@ class StatusesIndex < Chewy::Index end root date_detection: false do - field :id, type: 'long' - field :account_id, type: 'long' - - field :text, type: 'text', value: ->(status) { status.searchable_text } do - field :stemmed, type: 'text', analyzer: 'content' - end - - field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) } + field(:id, type: 'keyword') + field(:account_id, type: 'long') + field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') } + field(:searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }) + field(:language, type: 'keyword') + field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties }) + field(:created_at, type: 'date') end end diff --git a/app/controllers/api/v1/accounts/credentials_controller.rb b/app/controllers/api/v1/accounts/credentials_controller.rb index 7c7d70fd3..76ba75824 100644 --- a/app/controllers/api/v1/accounts/credentials_controller.rb +++ b/app/controllers/api/v1/accounts/credentials_controller.rb @@ -30,6 +30,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController :bot, :discoverable, :hide_collections, + :indexable, fields_attributes: [:name, :value] ) end diff --git a/app/controllers/settings/privacy_controller.rb b/app/controllers/settings/privacy_controller.rb index c2648eedd..1102c89fa 100644 --- a/app/controllers/settings/privacy_controller.rb +++ b/app/controllers/settings/privacy_controller.rb @@ -18,7 +18,7 @@ class Settings::PrivacyController < Settings::BaseController private def account_params - params.require(:account).permit(:discoverable, :unlocked, :show_collections, settings: UserSettings.keys) + params.require(:account).permit(:discoverable, :unlocked, :indexable, :show_collections, settings: UserSettings.keys) end def set_account diff --git a/app/lib/importer/public_statuses_index_importer.rb b/app/lib/importer/public_statuses_index_importer.rb new file mode 100644 index 000000000..8e36e36f9 --- /dev/null +++ b/app/lib/importer/public_statuses_index_importer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Importer::PublicStatusesIndexImporter < Importer::BaseImporter + def import! + indexable_statuses_scope.find_in_batches(batch_size: @batch_size) do |batch| + in_work_unit(batch.map(&:status_id)) do |status_ids| + bulk = ActiveRecord::Base.connection_pool.with_connection do + Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body + end + + indexed = 0 + deleted = 0 + + bulk.map! do |entry| + if entry[:index] + indexed += 1 + else + deleted += 1 + end + entry + end + + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) + + [indexed, deleted] + end + end + + wait! + end + + private + + def index + PublicStatusesIndex + end + + def indexable_statuses_scope + Status.indexable.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id') + end +end diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index aef05e9d9..dad99cbd2 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -36,7 +36,7 @@ class SearchQueryTransformer < Parslet::Transform def clause_to_filter(clause) case clause when PrefixClause - { term: { clause.filter => clause.term } } + { clause.type => { clause.filter => clause.term } } else raise "Unexpected clause type: #{clause}" end @@ -47,12 +47,10 @@ class SearchQueryTransformer < Parslet::Transform class << self def symbol(str) case str - when '+' + when '+', nil :must when '-' :must_not - when nil - :should else raise "Unknown operator: #{str}" end @@ -81,23 +79,52 @@ class SearchQueryTransformer < Parslet::Transform end class PrefixClause - attr_reader :filter, :operator, :term + attr_reader :type, :filter, :operator, :term def initialize(prefix, term) @operator = :filter + case prefix + when 'has', 'is' + @filter = :properties + @type = :term + @term = term + when 'language' + @filter = :language + @type = :term + @term = term when 'from' @filter = :account_id - - username, domain = term.gsub(/\A@/, '').split('@') - domain = nil if TagManager.instance.local_domain?(domain) - account = Account.find_remote!(username, domain) - - @term = account.id + @type = :term + @term = account_id_from_term(term) + when 'before' + @filter = :created_at + @type = :range + @term = { lt: term } + when 'after' + @filter = :created_at + @type = :range + @term = { gt: term } + when 'during' + @filter = :created_at + @type = :range + @term = { gte: term, lte: term } else raise Mastodon::SyntaxError end end + + private + + def account_id_from_term(term) + username, domain = term.gsub(/\A@/, '').split('@') + domain = nil if TagManager.instance.local_domain?(domain) + account = Account.find_remote(username, domain) + + # If the account is not found, we want to return empty results, so return + # an ID that does not exist + account&.id || -1 + end end rule(clause: subtree(:clause)) do diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb index 28c087b1c..ad1de0738 100644 --- a/app/lib/vacuum/statuses_vacuum.rb +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -20,7 +20,10 @@ class Vacuum::StatusesVacuum statuses.direct_visibility .includes(mentions: :account) .find_each(&:unlink_from_conversations!) - remove_from_search_index(statuses.ids) if Chewy.enabled? + if Chewy.enabled? + remove_from_index(statuses.ids, 'chewy:queue:StatusesIndex') + remove_from_index(statuses.ids, 'chewy:queue:PublicStatusesIndex') + end # Foreign keys take care of most associated records for us. # Media attachments will be orphaned. @@ -38,7 +41,7 @@ class Vacuum::StatusesVacuum Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false) end - def remove_from_search_index(status_ids) - with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', status_ids) } + def remove_from_index(status_ids, index) + with_redis { |redis| redis.sadd(index, status_ids) } end end diff --git a/app/models/account.rb b/app/models/account.rb index b1cb9eb5d..244f3da83 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -82,6 +82,7 @@ class Account < ApplicationRecord include DomainMaterializable include AccountMerging include AccountSearch + include AccountStatusesSearch enum protocol: { ostatus: 0, activitypub: 1 } enum suspension_origin: { local: 0, remote: 1 }, _prefix: true diff --git a/app/models/concerns/account_statuses_search.rb b/app/models/concerns/account_statuses_search.rb new file mode 100644 index 000000000..563a87105 --- /dev/null +++ b/app/models/concerns/account_statuses_search.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module AccountStatusesSearch + extend ActiveSupport::Concern + + included do + after_update_commit :enqueue_update_public_statuses_index, if: :saved_change_to_indexable? + after_destroy_commit :enqueue_remove_from_public_statuses_index, if: :indexable? + end + + def enqueue_update_public_statuses_index + if indexable? + enqueue_add_to_public_statuses_index + else + enqueue_remove_from_public_statuses_index + end + end + + def enqueue_add_to_public_statuses_index + return unless Chewy.enabled? + + AddToPublicStatusesIndexWorker.perform_async(id) + end + + def enqueue_remove_from_public_statuses_index + return unless Chewy.enabled? + + RemoveFromPublicStatusesIndexWorker.perform_async(id) + end + + def add_to_public_statuses_index! + return unless Chewy.enabled? + + statuses.indexable.find_in_batches do |batch| + PublicStatusesIndex.import(query: batch) + end + end + + def remove_from_public_statuses_index! + return unless Chewy.enabled? + + PublicStatusesIndex.filter(term: { account_id: id }).delete_all + end +end diff --git a/app/models/concerns/status_search_concern.rb b/app/models/concerns/status_search_concern.rb new file mode 100644 index 000000000..21048b568 --- /dev/null +++ b/app/models/concerns/status_search_concern.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module StatusSearchConcern + extend ActiveSupport::Concern + + included do + scope :indexable, -> { without_reblogs.where(visibility: :public).joins(:account).where(account: { indexable: true }) } + end + + def searchable_by(preloaded = nil) + ids = [] + + ids << account_id if local? + + if preloaded.nil? + ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id) + ids += favourites.joins(:account).merge(Account.local).pluck(:account_id) + ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id) + ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id) + ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present? + else + ids += preloaded.mentions[id] || [] + ids += preloaded.favourites[id] || [] + ids += preloaded.reblogs[id] || [] + ids += preloaded.bookmarks[id] || [] + ids += preloaded.votes[id] || [] + end + + ids.uniq + end + + def searchable_text + [ + spoiler_text, + FormattingHelper.extract_status_plain_text(self), + preloadable_poll&.options&.join("\n\n"), + ordered_media_attachments.map(&:description).join("\n\n"), + ].compact.join("\n\n") + end + + def searchable_properties + [].tap do |properties| + properties << 'image' if ordered_media_attachments.any?(&:image?) + properties << 'video' if ordered_media_attachments.any?(&:video?) + properties << 'audio' if ordered_media_attachments.any?(&:audio?) + properties << 'media' if with_media? + properties << 'poll' if with_poll? + properties << 'link' if with_preview_card? + properties << 'embed' if preview_cards.any?(&:video?) + properties << 'sensitive' if sensitive? + properties << 'reply' if reply? + end + end +end diff --git a/app/models/status.rb b/app/models/status.rb index 86fd8334a..760b8ec33 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -37,6 +37,7 @@ class Status < ApplicationRecord include StatusSnapshotConcern include RateLimitable include StatusSafeReblogInsert + include StatusSearchConcern rate_limit by: :account, family: :statuses @@ -47,6 +48,7 @@ class Status < ApplicationRecord attr_accessor :override_timestamps update_index('statuses', :proper) + update_index('public_statuses', :proper) enum visibility: { public: 0, unlisted: 1, private: 2, direct: 3, limited: 4 }, _suffix: :visibility @@ -165,37 +167,6 @@ class Status < ApplicationRecord "v3:#{super}" end - def searchable_by(preloaded = nil) - ids = [] - - ids << account_id if local? - - if preloaded.nil? - ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id) - ids += favourites.joins(:account).merge(Account.local).pluck(:account_id) - ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id) - ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id) - ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present? - else - ids += preloaded.mentions[id] || [] - ids += preloaded.favourites[id] || [] - ids += preloaded.reblogs[id] || [] - ids += preloaded.bookmarks[id] || [] - ids += preloaded.votes[id] || [] - end - - ids.uniq - end - - def searchable_text - [ - spoiler_text, - FormattingHelper.extract_status_plain_text(self), - preloadable_poll ? preloadable_poll.options.join("\n\n") : nil, - ordered_media_attachments.map(&:description).join("\n\n"), - ].compact.join("\n\n") - end - def to_log_human_identifier account.acct end @@ -270,6 +241,10 @@ class Status < ApplicationRecord preview_cards.any? end + def with_poll? + preloadable_poll.present? + end + def non_sensitive_with_media? !sensitive? && with_media? end diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index 4998d0039..31f39954f 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -8,13 +8,13 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer context_extensions :manually_approves_followers, :featured, :also_known_as, :moved_to, :property_value, :discoverable, :olm, :suspended, - :memorial + :memorial, :indexable attributes :id, :type, :following, :followers, :inbox, :outbox, :featured, :featured_tags, :preferred_username, :name, :summary, :url, :manually_approves_followers, - :discoverable, :published, :memorial + :discoverable, :indexable, :published, :memorial has_one :public_key, serializer: ActivityPub::PublicKeySerializer @@ -99,6 +99,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer object.suspended? ? false : (object.discoverable || false) end + def indexable + object.suspended? ? false : (object.indexable || false) + end + def name object.suspended? ? object.username : (object.display_name.presence || object.username) end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index f5cb339cd..c54cc1d35 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -35,7 +35,10 @@ class BatchedRemoveStatusService < BaseService # Since we skipped all callbacks, we also need to manually # deindex the statuses - Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) if Chewy.enabled? + if Chewy.enabled? + Chewy.strategy.current.update(StatusesIndex, statuses_and_reblogs) + Chewy.strategy.current.update(PublicStatusesIndex, statuses_and_reblogs) + end return if options[:skip_side_effects] diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 30937471b..4e1e7ea26 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -39,25 +39,15 @@ class SearchService < BaseService end def perform_statuses_search! - definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id })) - - definition = definition.filter(term: { account_id: @options[:account_id] }) if @options[:account_id].present? - - if @options[:min_id].present? || @options[:max_id].present? - range = {} - range[:gt] = @options[:min_id].to_i if @options[:min_id].present? - range[:lt] = @options[:max_id].to_i if @options[:max_id].present? - definition = definition.filter(range: { id: range }) - end - - results = definition.limit(@limit).offset(@offset).objects.compact - account_ids = results.map(&:account_id) - account_domains = results.map(&:account_domain) - preloaded_relations = @account.relations_map(account_ids, account_domains) - - results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? } - rescue Faraday::ConnectionFailed, Parslet::ParseFailed - [] + StatusesSearchService.new.call( + @query, + @account, + limit: @limit, + offset: @offset, + account_id: @options[:account_id], + min_id: @options[:min_id], + max_id: @options[:max_id] + ) end def perform_hashtags_search! @@ -114,8 +104,4 @@ class SearchService < BaseService def statuses_search? @options[:type].blank? || @options[:type] == 'statuses' end - - def parsed_query - SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query)) - end end diff --git a/app/services/statuses_search_service.rb b/app/services/statuses_search_service.rb new file mode 100644 index 000000000..21d6b71b7 --- /dev/null +++ b/app/services/statuses_search_service.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +class StatusesSearchService < BaseService + def call(query, account = nil, options = {}) + @query = query&.strip + @account = account + @options = options + @limit = options[:limit].to_i + @offset = options[:offset].to_i + + status_search_results + end + + private + + def status_search_results + definition = parsed_query.apply( + StatusesIndex.filter( + bool: { + should: [ + publicly_searchable, + non_publicly_searchable, + ], + + minimum_should_match: 1, + } + ) + ) + + # This is the best way to submit identical queries to multi-indexes though chewy + definition.instance_variable_get(:@parameters)[:indices].value[:indices] << PublicStatusesIndex + + results = definition.collapse(field: :id).order(_id: { order: :desc }).limit(@limit).offset(@offset).objects.compact + account_ids = results.map(&:account_id) + account_domains = results.map(&:account_domain) + preloaded_relations = @account.relations_map(account_ids, account_domains) + + results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? } + rescue Faraday::ConnectionFailed, Parslet::ParseFailed + [] + end + + def publicly_searchable + { + bool: { + must_not: { + exists: { + field: 'searchable_by', + }, + }, + }, + } + end + + def non_publicly_searchable + { + bool: { + must: [ + { + exists: { + field: 'searchable_by', + }, + }, + { + term: { searchable_by: @account.id }, + }, + ], + }, + } + end + + def parsed_query + SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query)) + end +end diff --git a/app/views/settings/privacy/show.html.haml b/app/views/settings/privacy/show.html.haml index ce31e60f0..3c1438258 100644 --- a/app/views/settings/privacy/show.html.haml +++ b/app/views/settings/privacy/show.html.haml @@ -24,6 +24,9 @@ %p.lead= t('privacy.search_hint_html') + .fields-group + = f.input :indexable, as: :boolean, wrapper: :with_label + = f.simple_fields_for :settings, current_user.settings do |ff| .fields-group = ff.input :indexable, wrapper: :with_label diff --git a/app/workers/add_to_public_statuses_index_worker.rb b/app/workers/add_to_public_statuses_index_worker.rb new file mode 100644 index 000000000..409e5e708 --- /dev/null +++ b/app/workers/add_to_public_statuses_index_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddToPublicStatusesIndexWorker + include Sidekiq::Worker + + def perform(account_id) + account = Account.find(account_id) + + return unless account.indexable? + + account.add_to_public_statuses_index! + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/remove_from_public_statuses_index_worker.rb b/app/workers/remove_from_public_statuses_index_worker.rb new file mode 100644 index 000000000..e108a5c20 --- /dev/null +++ b/app/workers/remove_from_public_statuses_index_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveFromPublicStatusesIndexWorker + include Sidekiq::Worker + + def perform(account_id) + account = Account.find(account_id) + + return if account.indexable? + + account.remove_from_public_statuses_index! + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index 2868a3b71..6c770d5a8 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -23,6 +23,6 @@ class Scheduler::IndexingScheduler end def indexes - [AccountsIndex, TagsIndex, StatusesIndex] + [AccountsIndex, TagsIndex, PublicStatusesIndex, StatusesIndex] end end diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 443b7617f..efda7b778 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -6,6 +6,7 @@ en: discoverable: Your public posts and profile may be featured or recommended in various areas of Mastodon and your profile may be suggested to other users. display_name: Your full name or your fun name. fields: Your homepage, pronouns, age, anything you want. + indexable: Your public posts may appear in search results on Mastodon. People who have interacted with your posts may be able to search them regardless. note: 'You can @mention other people or #hashtags.' show_collections: People will be able to browse through your follows and followers. People that you follow will see that you follow them regardless. unlocked: People will be able to follow you without requesting approval. Uncheck if you want to review follow requests and chose whether to accept or reject new followers. @@ -143,6 +144,7 @@ en: fields: name: Label value: Content + indexable: Include public posts in search results show_collections: Show follows and followers on profile unlocked: Automatically accept new followers account_alias: diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index 41862b5b6..481e01d8e 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -10,6 +10,7 @@ module Mastodon::CLI InstancesIndex, AccountsIndex, TagsIndex, + PublicStatusesIndex, StatusesIndex, ].freeze diff --git a/spec/chewy/public_statuses_index_spec.rb b/spec/chewy/public_statuses_index_spec.rb new file mode 100644 index 000000000..2f93d0ff0 --- /dev/null +++ b/spec/chewy/public_statuses_index_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe PublicStatusesIndex do + describe 'Searching the index' do + before do + mock_elasticsearch_response(described_class, raw_response) + end + + it 'returns results from a query' do + results = described_class.query(match: { name: 'status' }) + + expect(results).to eq [] + end + end + + def raw_response + { + took: 3, + hits: { + hits: [ + { + _id: '0', + _score: 1.6375021, + }, + ], + }, + } + end +end diff --git a/spec/lib/importer/public_statuses_index_importer_spec.rb b/spec/lib/importer/public_statuses_index_importer_spec.rb new file mode 100644 index 000000000..bc7c038a9 --- /dev/null +++ b/spec/lib/importer/public_statuses_index_importer_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Importer::PublicStatusesIndexImporter do + describe 'import!' do + let(:pool) { Concurrent::FixedThreadPool.new(5) } + let(:importer) { described_class.new(batch_size: 123, executor: pool) } + + before { Fabricate(:status, account: Fabricate(:account, indexable: true)) } + + it 'indexes relevant statuses' do + expect { importer.import! }.to update_index(PublicStatusesIndex) + end + end +end diff --git a/spec/lib/search_query_transformer_spec.rb b/spec/lib/search_query_transformer_spec.rb index 109533469..953f9acb2 100644 --- a/spec/lib/search_query_transformer_spec.rb +++ b/spec/lib/search_query_transformer_spec.rb @@ -9,8 +9,8 @@ describe SearchQueryTransformer do it 'sets attributes' do transformer = described_class.new.apply(parser) - expect(transformer.should_clauses.first).to be_a(SearchQueryTransformer::TermClause) - expect(transformer.must_clauses.first).to be_nil + expect(transformer.should_clauses.first).to be_nil + expect(transformer.must_clauses.first).to be_a(SearchQueryTransformer::TermClause) expect(transformer.must_not_clauses.first).to be_nil expect(transformer.filter_clauses.first).to be_nil end diff --git a/spec/models/concerns/account_statuses_search_spec.rb b/spec/models/concerns/account_statuses_search_spec.rb new file mode 100644 index 000000000..46362936f --- /dev/null +++ b/spec/models/concerns/account_statuses_search_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe AccountStatusesSearch do + let(:account) { Fabricate(:account, indexable: indexable) } + + before do + allow(Chewy).to receive(:enabled?).and_return(true) + end + + describe '#enqueue_update_public_statuses_index' do + before do + allow(account).to receive(:enqueue_add_to_public_statuses_index) + allow(account).to receive(:enqueue_remove_from_public_statuses_index) + end + + context 'when account is indexable' do + let(:indexable) { true } + + it 'enqueues add_to_public_statuses_index and not to remove_from_public_statuses_index' do + account.enqueue_update_public_statuses_index + expect(account).to have_received(:enqueue_add_to_public_statuses_index) + expect(account).to_not have_received(:enqueue_remove_from_public_statuses_index) + end + end + + context 'when account is not indexable' do + let(:indexable) { false } + + it 'enqueues remove_from_public_statuses_index and not to add_to_public_statuses_index' do + account.enqueue_update_public_statuses_index + expect(account).to have_received(:enqueue_remove_from_public_statuses_index) + expect(account).to_not have_received(:enqueue_add_to_public_statuses_index) + end + end + end + + describe '#enqueue_add_to_public_statuses_index' do + let(:indexable) { true } + let(:worker) { AddToPublicStatusesIndexWorker } + + before do + allow(worker).to receive(:perform_async) + end + + it 'enqueues AddToPublicStatusesIndexWorker' do + account.enqueue_add_to_public_statuses_index + expect(worker).to have_received(:perform_async).with(account.id) + end + end + + describe '#enqueue_remove_from_public_statuses_index' do + let(:indexable) { false } + let(:worker) { RemoveFromPublicStatusesIndexWorker } + + before do + allow(worker).to receive(:perform_async) + end + + it 'enqueues RemoveFromPublicStatusesIndexWorker' do + account.enqueue_remove_from_public_statuses_index + expect(worker).to have_received(:perform_async).with(account.id) + end + end +end diff --git a/spec/workers/add_to_public_statuses_index_worker_spec.rb b/spec/workers/add_to_public_statuses_index_worker_spec.rb new file mode 100644 index 000000000..fa1507224 --- /dev/null +++ b/spec/workers/add_to_public_statuses_index_worker_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe AddToPublicStatusesIndexWorker do + describe '#perform' do + let(:account) { Fabricate(:account, indexable: indexable) } + let(:account_id) { account.id } + + before do + allow(Account).to receive(:find).with(account_id).and_return(account) unless account.nil? + allow(account).to receive(:add_to_public_statuses_index!) unless account.nil? + end + + context 'when account is indexable' do + let(:indexable) { true } + + it 'adds the account to the public statuses index' do + subject.perform(account_id) + expect(account).to have_received(:add_to_public_statuses_index!) + end + end + + context 'when account is not indexable' do + let(:indexable) { false } + + it 'does not add the account to public statuses index' do + subject.perform(account_id) + expect(account).to_not have_received(:add_to_public_statuses_index!) + end + end + + context 'when account does not exist' do + let(:account) { nil } + let(:account_id) { 999 } + + it 'does not raise an error' do + expect { subject.perform(account_id) }.to_not raise_error + end + end + end +end diff --git a/spec/workers/remove_from_public_statuses_index_worker_spec.rb b/spec/workers/remove_from_public_statuses_index_worker_spec.rb new file mode 100644 index 000000000..43ff211ea --- /dev/null +++ b/spec/workers/remove_from_public_statuses_index_worker_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe RemoveFromPublicStatusesIndexWorker do + describe '#perform' do + let(:account) { Fabricate(:account, indexable: indexable) } + let(:account_id) { account.id } + + before do + allow(Account).to receive(:find).with(account_id).and_return(account) unless account.nil? + allow(account).to receive(:remove_from_public_statuses_index!) unless account.nil? + end + + context 'when account is not indexable' do + let(:indexable) { false } + + it 'removes the account from public statuses index' do + subject.perform(account_id) + expect(account).to have_received(:remove_from_public_statuses_index!) + end + end + + context 'when account is indexable' do + let(:indexable) { true } + + it 'does not remove the account from public statuses index' do + subject.perform(account_id) + expect(account).to_not have_received(:remove_from_public_statuses_index!) + end + end + + context 'when account does not exist' do + let(:account) { nil } + let(:account_id) { 999 } + + it 'does not raise an error' do + expect { subject.perform(account_id) }.to_not raise_error + end + end + end +end From 389b7d23dbd32fa0ab563083d76b8aaf0d6137da Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 24 Aug 2023 21:07:39 +0200 Subject: [PATCH 02/10] Fix changelog wording and missing items (#26638) --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 107dfaca3..fe66adc08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,7 +101,7 @@ The following changelog entries focus on changes visible to users, administrator - **Change translation feature to cover Content Warnings, poll options and media descriptions** ([c960657](https://github.com/mastodon/mastodon/pull/24175), [S-H-GAMELINKS](https://github.com/mastodon/mastodon/pull/25251), [c960657](https://github.com/mastodon/mastodon/pull/26168), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26452)) - **Change account search to match by text when opted-in** ([jsgoldstein](https://github.com/mastodon/mastodon/pull/25599), [Gargron](https://github.com/mastodon/mastodon/pull/26378)) - **Change import feature to be clearer, less error-prone and more reliable** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21054), [mgmn](https://github.com/mastodon/mastodon/pull/24874)) -- **Change local and federated timelines to be in a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247)) +- **Change local and federated timelines to be tabs of a single “Live feeds” column** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25641), [Gargron](https://github.com/mastodon/mastodon/pull/25683), [mgmn](https://github.com/mastodon/mastodon/pull/25694), [Plastikmensch](https://github.com/mastodon/mastodon/pull/26247)) - **Change user archive export to be faster and more reliable, and export `.zip` archives instead of `.tar.gz` ones** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23360), [TheEssem](https://github.com/mastodon/mastodon/pull/25034)) - **Change `mastodon-streaming` systemd unit files to be templated** ([e-nomem](https://github.com/mastodon/mastodon/pull/24751)) - **Change `statsd` integration to disable sidekiq metrics by default** ([mjankowski](https://github.com/mastodon/mastodon/pull/25265), [mjankowski](https://github.com/mastodon/mastodon/pull/25336), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/26310)) @@ -189,6 +189,7 @@ The following changelog entries focus on changes visible to users, administrator - **Fix log-in flow when involving both OAuth and external authentication** ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24073)) - **Fix broken links in account gallery** ([c960657](https://github.com/mastodon/mastodon/pull/24218)) - **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392)) +- **Fix migration handler not updating lists** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24808)) - Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500)) - Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409)) - Fix adding column with default value taking longer on Postgres >= 11 ([Gargron](https://github.com/mastodon/mastodon/pull/26375)) From c66f7565226e6ad37d23fd4b89f99f0f277d83a4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 25 Aug 2023 10:08:44 +0200 Subject: [PATCH 03/10] Update babel monorepo to v7.22.11 (#26640) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 129 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 86 insertions(+), 43 deletions(-) diff --git a/yarn.lock b/yarn.lock index 216e22f68..6894f4388 100644 --- a/yarn.lock +++ b/yarn.lock @@ -50,24 +50,24 @@ integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== "@babel/core@^7.10.4", "@babel/core@^7.11.1", "@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.22.1": - version "7.22.10" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.10.tgz#aad442c7bcd1582252cb4576747ace35bc122f35" - integrity sha512-fTmqbbUBAwCcre6zPzNngvsI0aNrPZe77AeqvDxWM9Nm+04RrJ3CAmGHA9f7lJQY6ZMhRztNemy4uslDxTX4Qw== + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.11.tgz#8033acaa2aa24c3f814edaaa057f3ce0ba559c24" + integrity sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ== dependencies: "@ampproject/remapping" "^2.2.0" "@babel/code-frame" "^7.22.10" "@babel/generator" "^7.22.10" "@babel/helper-compilation-targets" "^7.22.10" "@babel/helper-module-transforms" "^7.22.9" - "@babel/helpers" "^7.22.10" - "@babel/parser" "^7.22.10" + "@babel/helpers" "^7.22.11" + "@babel/parser" "^7.22.11" "@babel/template" "^7.22.5" - "@babel/traverse" "^7.22.10" - "@babel/types" "^7.22.10" + "@babel/traverse" "^7.22.11" + "@babel/types" "^7.22.11" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" - json5 "^2.2.2" + json5 "^2.2.3" semver "^6.3.1" "@babel/generator@^7.22.10", "@babel/generator@^7.22.5", "@babel/generator@^7.7.2": @@ -113,6 +113,21 @@ lru-cache "^5.1.1" semver "^6.3.1" +"@babel/helper-create-class-features-plugin@^7.22.11": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.22.11.tgz#4078686740459eeb4af3494a273ac09148dfb213" + integrity sha512-y1grdYL4WzmUDBRGK0pDbIoFd7UZKoDurDzWEoNMYoj1EL+foGRQNyPWDcC+YyegN5y1DUsFFmzjGijB3nSVAQ== + dependencies: + "@babel/helper-annotate-as-pure" "^7.22.5" + "@babel/helper-environment-visitor" "^7.22.5" + "@babel/helper-function-name" "^7.22.5" + "@babel/helper-member-expression-to-functions" "^7.22.5" + "@babel/helper-optimise-call-expression" "^7.22.5" + "@babel/helper-replace-supers" "^7.22.9" + "@babel/helper-skip-transparent-expression-wrappers" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + semver "^6.3.1" + "@babel/helper-create-class-features-plugin@^7.22.5": version "7.22.10" resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.22.10.tgz#dd2612d59eac45588021ac3d6fa976d08f4e95a3" @@ -268,14 +283,14 @@ "@babel/template" "^7.22.5" "@babel/types" "^7.22.10" -"@babel/helpers@^7.22.10": - version "7.22.10" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.10.tgz#ae6005c539dfbcb5cd71fb51bfc8a52ba63bc37a" - integrity sha512-a41J4NW8HyZa1I1vAndrraTlPZ/eZoga2ZgS7fEr0tZJGVU4xqdE80CEm0CcNjha5EZ8fTBYLKHF0kqDUuAwQw== +"@babel/helpers@^7.22.11": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.11.tgz#b02f5d5f2d7abc21ab59eeed80de410ba70b056a" + integrity sha512-vyOXC8PBWaGc5h7GMsNx68OH33cypkEDJCHvYVVgVbbxJDROYVtexSk0gK5iCF1xNjRIN2s8ai7hwkWDq5szWg== dependencies: "@babel/template" "^7.22.5" - "@babel/traverse" "^7.22.10" - "@babel/types" "^7.22.10" + "@babel/traverse" "^7.22.11" + "@babel/types" "^7.22.11" "@babel/highlight@^7.22.10", "@babel/highlight@^7.22.5": version "7.22.10" @@ -286,11 +301,16 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.10", "@babel/parser@^7.22.5": +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7": version "7.22.10" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.10.tgz#e37634f9a12a1716136c44624ef54283cabd3f55" integrity sha512-lNbdGsQb9ekfsnjFGhEiF4hfFqGgfOP3H3d27re3n+CGhNuTSUEQdfWk556sTLNTloczcdM5TYF2LhzmDQKyvQ== +"@babel/parser@^7.22.11", "@babel/parser@^7.22.5": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.11.tgz#becf8ee33aad2a35ed5607f521fe6e72a615f905" + integrity sha512-R5zb8eJIBPJriQtbH/htEQy4k7E2dHWlD2Y2VT07JCzwYZHBxV5ZYtM0UhXSNMT74LyxuM+b1jdL7pSesXbC/g== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.22.5.tgz#87245a21cd69a73b0b81bcda98d443d6df08f05e" @@ -640,6 +660,15 @@ "@babel/helper-module-transforms" "^7.22.5" "@babel/helper-plugin-utils" "^7.22.5" +"@babel/plugin-transform-modules-commonjs@^7.22.11": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.22.11.tgz#d7991d3abad199c03b68ee66a64f216c47ffdfae" + integrity sha512-o2+bg7GDS60cJMgz9jWqRUsWkMzLCxp+jFDeDUT5sjRlAxcJWZ2ylNdI7QQ2+CH5hWu7OnN+Cv3htt7AkSf96g== + dependencies: + "@babel/helper-module-transforms" "^7.22.9" + "@babel/helper-plugin-utils" "^7.22.5" + "@babel/helper-simple-access" "^7.22.5" + "@babel/plugin-transform-modules-commonjs@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.22.5.tgz#7d9875908d19b8c0536085af7b053fd5bd651bfa" @@ -683,9 +712,9 @@ "@babel/helper-plugin-utils" "^7.22.5" "@babel/plugin-transform-nullish-coalescing-operator@^7.22.3", "@babel/plugin-transform-nullish-coalescing-operator@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.22.5.tgz#f8872c65776e0b552e0849d7596cddd416c3e381" - integrity sha512-6CF8g6z1dNYZ/VXok5uYkkBBICHZPiGEl7oDnAx2Mt1hlHVHOSIKWJaXHjQJA5VB43KZnXZDIexMchY4y2PGdA== + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.22.11.tgz#debef6c8ba795f5ac67cd861a81b744c5d38d9fc" + integrity sha512-YZWOw4HxXrotb5xsjMJUDlLgcDXSfO9eCmdl1bgW4+/lAGdkjaEvOnQ4p5WKKdUgSzO39dgPl0pTnfxm0OAXcg== dependencies: "@babel/helper-plugin-utils" "^7.22.5" "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" @@ -877,13 +906,13 @@ dependencies: "@babel/helper-plugin-utils" "^7.22.5" -"@babel/plugin-transform-typescript@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.22.5.tgz#5c0f7adfc1b5f38c4dbc8f79b1f0f8074134bd7d" - integrity sha512-SMubA9S7Cb5sGSFFUlqxyClTA9zWJ8qGQrppNUm05LtFuN1ELRFNndkix4zUJrC9F+YivWwa1dHMSyo0e0N9dA== +"@babel/plugin-transform-typescript@^7.22.11": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.22.11.tgz#9f27fb5e51585729374bb767ab6a6d9005a23329" + integrity sha512-0E4/L+7gfvHub7wsbTv03oRtD69X31LByy44fGmFzbZScpupFByMcgCJ0VbBTkzyjSJKuRoGN8tcijOWKTmqOA== dependencies: "@babel/helper-annotate-as-pure" "^7.22.5" - "@babel/helper-create-class-features-plugin" "^7.22.5" + "@babel/helper-create-class-features-plugin" "^7.22.11" "@babel/helper-plugin-utils" "^7.22.5" "@babel/plugin-syntax-typescript" "^7.22.5" @@ -1026,15 +1055,15 @@ "@babel/plugin-transform-react-pure-annotations" "^7.22.5" "@babel/preset-typescript@^7.21.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.22.5.tgz#16367d8b01d640e9a507577ed4ee54e0101e51c8" - integrity sha512-YbPaal9LxztSGhmndR46FmAbkJ/1fAsw293tSU+I5E5h+cnJ3d4GTwyUgGYmOXJYdGA+uNePle4qbaRzj2NISQ== + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.22.11.tgz#f218cd0345524ac888aa3dc32f029de5b064b575" + integrity sha512-tWY5wyCZYBGY7IlalfKI1rLiGlIfnwsRHZqlky0HVv8qviwQ1Uo/05M6+s+TcTCVa6Bmoo2uJW5TMFX6Wa4qVg== dependencies: "@babel/helper-plugin-utils" "^7.22.5" "@babel/helper-validator-option" "^7.22.5" "@babel/plugin-syntax-jsx" "^7.22.5" - "@babel/plugin-transform-modules-commonjs" "^7.22.5" - "@babel/plugin-transform-typescript" "^7.22.5" + "@babel/plugin-transform-modules-commonjs" "^7.22.11" + "@babel/plugin-transform-typescript" "^7.22.11" "@babel/regjsgen@^0.8.0": version "0.8.0" @@ -1049,9 +1078,9 @@ regenerator-runtime "^0.12.0" "@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.0", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.2.0", "@babel/runtime@^7.20.13", "@babel/runtime@^7.20.7", "@babel/runtime@^7.22.3", "@babel/runtime@^7.3.1", "@babel/runtime@^7.5.5", "@babel/runtime@^7.6.3", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": - version "7.22.10" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.10.tgz#ae3e9631fd947cb7e3610d3e9d8fef5f76696682" - integrity sha512-21t/fkKLMZI4pqP2wlmsQAWnYW1PDyKyyUV4vCi+B25ydmdaYTKXPwCj0BzSUnZf4seIiYvSA3jcZ3gdsMFkLQ== + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.11.tgz#7a9ba3bbe406ad6f9e8dd4da2ece453eb23a77a4" + integrity sha512-ee7jVNlWN09+KftVOu9n7S8gQzD/Z6hN/I8VBRXW4P1+Xe7kJGXMwu8vds4aGIMHZnNbdpSWCfZZtinytpcAvA== dependencies: regenerator-runtime "^0.14.0" @@ -1080,10 +1109,10 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/traverse@^7.22.10": - version "7.22.10" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.10.tgz#20252acb240e746d27c2e82b4484f199cf8141aa" - integrity sha512-Q/urqV4pRByiNNpb/f5OSv28ZlGJiFiiTh+GAHktbIrkPhPbl90+uW6SmpoLyZqutrg9AEaEf3Q/ZBRHBXgxig== +"@babel/traverse@^7.22.11": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c" + integrity sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ== dependencies: "@babel/code-frame" "^7.22.10" "@babel/generator" "^7.22.10" @@ -1091,12 +1120,12 @@ "@babel/helper-function-name" "^7.22.5" "@babel/helper-hoist-variables" "^7.22.5" "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.10" - "@babel/types" "^7.22.10" + "@babel/parser" "^7.22.11" + "@babel/types" "^7.22.11" debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.10", "@babel/types@^7.22.5", "@babel/types@^7.3.3", "@babel/types@^7.4.4": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.3.3", "@babel/types@^7.4.4": version "7.22.10" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.10.tgz#4a9e76446048f2c66982d1a989dd12b8a2d2dc03" integrity sha512-obaoigiLrlDZ7TUQln/8m4mSqIW2QFeOrCQc9r+xsaHGNoplVNYlRVpsfE8Vj35GEm2ZH4ZhrNYogs/3fj85kg== @@ -1114,6 +1143,15 @@ "@babel/helper-validator-identifier" "^7.22.5" to-fast-properties "^2.0.0" +"@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.22.5": + version "7.22.11" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.11.tgz#0e65a6a1d4d9cbaa892b2213f6159485fe632ea2" + integrity sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.5" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -3901,7 +3939,12 @@ caniuse-lite@^1.0.30001502: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001515.tgz#418aefeed9d024cd3129bfae0ccc782d4cb8f12b" integrity sha512-eEFDwUOZbE24sb+Ecsx3+OvNETqjWIdabMy52oOkIgcUtAsQifjUG9q4U9dgTHJM2mfk4uEPxc0+xuFdJ629QA== -caniuse-lite@^1.0.30001517, caniuse-lite@^1.0.30001520: +caniuse-lite@^1.0.30001517: + version "1.0.30001522" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001522.tgz#44b87a406c901269adcdb834713e23582dd71856" + integrity sha512-TKiyTVZxJGhsTszLuzb+6vUZSjVOAhClszBr2Ta2k9IwtNBT/4dzmL6aywt0HCgEZlmwJzXJd8yNiob6HgwTRg== + +caniuse-lite@^1.0.30001520: version "1.0.30001520" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001520.tgz#62e2b7a1c7b35269594cf296a80bdf8cb9565006" integrity sha512-tahF5O9EiiTzwTUqAeFjIZbn4Dnqxzz7ktrgGlMYNLH43Ul26IgTMH/zvL3DG0lZxBYnlT04axvInszUsZULdA== @@ -5050,9 +5093,9 @@ electron-to-chromium@^1.4.428: integrity sha512-/g3UyNDmDd6ebeWapmAoiyy+Sy2HyJ+/X8KyvNeHfKRFfHaA2W8oF5fxD5F3tjBDcjpwo0iek6YNgxNXDBoEtA== electron-to-chromium@^1.4.477: - version "1.4.490" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.490.tgz#d99286f6e915667fa18ea4554def1aa60eb4d5f1" - integrity sha512-6s7NVJz+sATdYnIwhdshx/N/9O6rvMxmhVoDSDFdj6iA45gHR8EQje70+RYsF4GeB+k0IeNSBnP7yG9ZXJFr7A== + version "1.4.500" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.500.tgz#7dd05fdfbe02ed34b9f6099cfe01407b473d5af7" + integrity sha512-P38NO8eOuWOKY1sQk5yE0crNtrjgjJj6r3NrbIKtG18KzCHmHE2Bt+aQA7/y0w3uYsHWxDa6icOohzjLJ4vJ4A== elliptic@^6.5.3: version "6.5.4" @@ -7844,7 +7887,7 @@ json5@^1.0.1, json5@^1.0.2: dependencies: minimist "^1.2.0" -json5@^2.1.2, json5@^2.2.0, json5@^2.2.2: +json5@^2.1.2, json5@^2.2.0, json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== From c3a42e1280759ea28bee11f241aef892b148bf6a Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Aug 2023 11:31:20 +0200 Subject: [PATCH 04/10] Add `data-nosnippet` attribute to remote posts and local posts with `noindex` (#26648) --- app/javascript/mastodon/components/status.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/components/status.jsx b/app/javascript/mastodon/components/status.jsx index 30692d1cd..e1728910e 100644 --- a/app/javascript/mastodon/components/status.jsx +++ b/app/javascript/mastodon/components/status.jsx @@ -550,7 +550,7 @@ class Status extends ImmutablePureComponent { return ( -
+
{prepend}
From e3825a13c124b9d2b2af221ef6bbe8ced3e8c3c3 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Aug 2023 12:01:55 +0200 Subject: [PATCH 05/10] Add PublicStatusesCheck to Elasticsearch index check on admin dashboard (#26650) --- app/lib/admin/system_check/elasticsearch_check.rb | 1 + spec/lib/admin/system_check/elasticsearch_check_spec.rb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 91070756c..2092ef0dd 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -6,6 +6,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck AccountsIndex, TagsIndex, StatusesIndex, + PublicStatusesIndex, ].freeze def skip? diff --git a/spec/lib/admin/system_check/elasticsearch_check_spec.rb b/spec/lib/admin/system_check/elasticsearch_check_spec.rb index f3918d403..a885640ce 100644 --- a/spec/lib/admin/system_check/elasticsearch_check_spec.rb +++ b/spec/lib/admin/system_check/elasticsearch_check_spec.rb @@ -17,6 +17,7 @@ describe Admin::SystemCheck::ElasticsearchCheck do allow(Chewy.client.indices).to receive_messages(get_mapping: { AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys, StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys, + PublicStatusesIndex.index_name => PublicStatusesIndex.mappings_hash.deep_stringify_keys, InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys, TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys, }, get_settings: { @@ -90,6 +91,7 @@ describe Admin::SystemCheck::ElasticsearchCheck do allow(Chewy.client.indices).to receive(:get_mapping).and_return({ AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys, StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys, + PublicStatusesIndex.index_name => PublicStatusesIndex.mappings_hash.deep_stringify_keys, InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys, TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys, }) From f2ec2876a45133668154a757d53283e02ecb20d5 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Aug 2023 12:02:09 +0200 Subject: [PATCH 06/10] =?UTF-8?q?Add=20Elasticsearch/OpenSearch=20version?= =?UTF-8?q?=20to=20=E2=80=9CSoftware=E2=80=9D=20in=20admin=20dashboard=20(?= =?UTF-8?q?#26652)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dimension/software_versions_dimension.rb | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb index 9ab3776c9..72a98a88a 100644 --- a/app/lib/admin/metrics/dimension/software_versions_dimension.rb +++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb @@ -10,7 +10,7 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim protected def perform_query - [mastodon_version, ruby_version, postgresql_version, redis_version] + [mastodon_version, ruby_version, postgresql_version, redis_version, elasticsearch_version].compact end def mastodon_version @@ -57,6 +57,22 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim } end + def elasticsearch_version + return unless Chewy.enabled? + + client_info = Chewy.client.info + version = client_info.dig('version', 'number') + + { + key: 'elasticsearch', + human_key: client_info.dig('version', 'distribution') == 'opensearch' ? 'OpenSearch' : 'Elasticsearch', + value: version, + human_value: version, + } + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error + nil + end + def redis_info @redis_info ||= if redis.is_a?(Redis::Namespace) redis.redis.info From 82ec6f162b617e9c46f23ebea1cf27e1e039121c Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Aug 2023 17:01:56 +0200 Subject: [PATCH 07/10] Fix statuses search Elasticsearch query (#26657) --- app/services/statuses_search_service.rb | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/app/services/statuses_search_service.rb b/app/services/statuses_search_service.rb index 21d6b71b7..2a4e533cf 100644 --- a/app/services/statuses_search_service.rb +++ b/app/services/statuses_search_service.rb @@ -15,7 +15,7 @@ class StatusesSearchService < BaseService def status_search_results definition = parsed_query.apply( - StatusesIndex.filter( + Chewy::Search::Request.new(StatusesIndex, PublicStatusesIndex).filter( bool: { should: [ publicly_searchable, @@ -27,9 +27,6 @@ class StatusesSearchService < BaseService ) ) - # This is the best way to submit identical queries to multi-indexes though chewy - definition.instance_variable_get(:@parameters)[:indices].value[:indices] << PublicStatusesIndex - results = definition.collapse(field: :id).order(_id: { order: :desc }).limit(@limit).offset(@offset).objects.compact account_ids = results.map(&:account_id) account_domains = results.map(&:account_domain) @@ -42,13 +39,7 @@ class StatusesSearchService < BaseService def publicly_searchable { - bool: { - must_not: { - exists: { - field: 'searchable_by', - }, - }, - }, + term: { _index: PublicStatusesIndex.index_name }, } end @@ -57,9 +48,7 @@ class StatusesSearchService < BaseService bool: { must: [ { - exists: { - field: 'searchable_by', - }, + term: { _index: StatusesIndex.index_name }, }, { term: { searchable_by: @account.id }, From e4c0ce18a32cfc8a911a4fcd3fd0b310755f609c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 25 Aug 2023 17:02:33 +0200 Subject: [PATCH 08/10] Update dependency immutable to v4.3.4 (#26655) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6894f4388..a630a752e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6686,9 +6686,9 @@ immutable@^3.8.2: integrity sha512-15gZoQ38eYjEjxkorfbcgBKBL6R7T459OuK+CpcWt7O3KF4uPCx2tD0uFETlUDIyo+1789crbMhTvQBSR5yBMg== immutable@^4.0.0, immutable@^4.0.0-rc.1, immutable@^4.3.0: - version "4.3.3" - resolved "https://registry.yarnpkg.com/immutable/-/immutable-4.3.3.tgz#8934ff6826d996a7642c8dc4b46e694dd19561e3" - integrity sha512-808ZFYMsIRAjLAu5xkKo0TsbY9LBy9H5MazTKIEHerNkg0ymgilGfBPMR/3G7d/ihGmuK2Hw8S1izY2d3kd3wA== + version "4.3.4" + resolved "https://registry.yarnpkg.com/immutable/-/immutable-4.3.4.tgz#2e07b33837b4bb7662f288c244d1ced1ef65a78f" + integrity sha512-fsXeu4J4i6WNWSikpI88v/PcVflZz+6kMhUfIwc5SY+poQRPnaf5V7qds6SUyUN3cVxEzuCab7QIoLOQ+DQ1wA== import-fresh@^3.2.1: version "3.3.0" From 8e8747c564ce3f06a96fc031cd0da9297d631fe0 Mon Sep 17 00:00:00 2001 From: jsgoldstein Date: Fri, 25 Aug 2023 12:16:08 -0400 Subject: [PATCH 09/10] Update ordering to use `id` from body of document instead of deprecated `_id` (#26659) --- app/services/statuses_search_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/statuses_search_service.rb b/app/services/statuses_search_service.rb index 2a4e533cf..0d0de2a9d 100644 --- a/app/services/statuses_search_service.rb +++ b/app/services/statuses_search_service.rb @@ -27,7 +27,7 @@ class StatusesSearchService < BaseService ) ) - results = definition.collapse(field: :id).order(_id: { order: :desc }).limit(@limit).offset(@offset).objects.compact + results = definition.collapse(field: :id).order(id: { order: :desc }).limit(@limit).offset(@offset).objects.compact account_ids = results.map(&:account_id) account_domains = results.map(&:account_domain) preloaded_relations = @account.relations_map(account_ids, account_domains) From 072112867b9a3ec090ad2c92d6363b47b2265d74 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Aug 2023 18:17:22 +0200 Subject: [PATCH 10/10] Fix dashboard check for Elasticsearch suggested command including incorrect names (#26658) --- app/lib/admin/system_check/elasticsearch_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 2092ef0dd..c0a1a21e8 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -86,7 +86,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def mismatched_indexes @mismatched_indexes ||= INDEXES.filter_map do |klass| - klass.index_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash + klass.base_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash end end