From 2f932cb2bb9add10014181d978331efcf61d30f5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Aug 2023 01:51:10 +0200 Subject: [PATCH 1/5] Add client-side timeout on resend confirmation button (#26300) --- app/javascript/packs/sign_up.js | 26 ++++++++++++++++++++++++++ app/views/auth/setup/show.html.haml | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/app/javascript/packs/sign_up.js b/app/javascript/packs/sign_up.js index 9aae9c11b..cf9c83777 100644 --- a/app/javascript/packs/sign_up.js +++ b/app/javascript/packs/sign_up.js @@ -13,4 +13,30 @@ ready(() => { console.error(error); }); }, 5000); + + document.querySelectorAll('.timer-button').forEach(button => { + let counter = 30; + + const container = document.createElement('span'); + + const updateCounter = () => { + container.innerText = ` (${counter})`; + }; + + updateCounter(); + + const countdown = setInterval(() => { + counter--; + + if (counter === 0) { + button.disabled = false; + button.removeChild(container); + clearInterval(countdown); + } else { + updateCounter(); + } + }, 1000); + + button.appendChild(container); + }); }); diff --git a/app/views/auth/setup/show.html.haml b/app/views/auth/setup/show.html.haml index 64deca334..97c826d70 100644 --- a/app/views/auth/setup/show.html.haml +++ b/app/views/auth/setup/show.html.haml @@ -19,6 +19,6 @@ = f.input :email, required: true, hint: false, input_html: { 'aria-label': t('simple_form.labels.defaults.email'), autocomplete: 'off' } .actions - = f.submit t('auth.resend_confirmation'), class: 'button' + = f.button :button, t('auth.resend_confirmation'), type: :submit, class: 'button timer-button', disabled: true .form-footer= render 'auth/shared/links' From f55f0ab0c35e13744f05d6643600e6907b24226a Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Thu, 3 Aug 2023 10:05:35 +0200 Subject: [PATCH 2/5] Allow spaces around commas in ALLOWED_PRIVATE_ADDRESSES (#26297) --- app/lib/request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/request.rb b/app/lib/request.rb index adc9a48f3..e5a9476a8 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -346,7 +346,7 @@ class Request end def private_address_exceptions - @private_address_exceptions = (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(',').map { |addr| IPAddr.new(addr) } + @private_address_exceptions = (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(/(?:\s*,\s*|\s+)/).map { |addr| IPAddr.new(addr) } end end end From a0fad5c8bb0eaf9fea619f94d102045f8c589c30 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Aug 2023 11:04:05 +0200 Subject: [PATCH 3/5] Change indexing frequency from 5 minutes to 1 minute, add locks to schedulers (#26304) --- app/workers/scheduler/follow_recommendations_scheduler.rb | 2 +- app/workers/scheduler/indexing_scheduler.rb | 6 ++---- app/workers/scheduler/instance_refresh_scheduler.rb | 2 +- app/workers/scheduler/ip_cleanup_scheduler.rb | 2 +- app/workers/scheduler/pghero_scheduler.rb | 2 +- app/workers/scheduler/scheduled_statuses_scheduler.rb | 2 +- app/workers/scheduler/suspended_user_cleanup_scheduler.rb | 2 +- app/workers/scheduler/user_cleanup_scheduler.rb | 2 +- app/workers/scheduler/vacuum_scheduler.rb | 2 +- config/sidekiq.yml | 2 +- 10 files changed, 11 insertions(+), 13 deletions(-) diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb index 17cf3f2cc..ea59ad986 100644 --- a/app/workers/scheduler/follow_recommendations_scheduler.rb +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -4,7 +4,7 @@ class Scheduler::FollowRecommendationsScheduler include Sidekiq::Worker include Redisable - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i # The maximum number of accounts that can be requested in one page from the # API is 80, and the suggestions API does not allow pagination. This number diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index d622f5586..2868a3b71 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -4,7 +4,7 @@ class Scheduler::IndexingScheduler include Sidekiq::Worker include Redisable - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i IMPORT_BATCH_SIZE = 1000 SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE @@ -16,9 +16,7 @@ class Scheduler::IndexingScheduler with_redis do |redis| redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| type.import!(ids) - redis.pipelined do |pipeline| - pipeline.srem("chewy:queue:#{type.name}", ids) - end + redis.srem("chewy:queue:#{type.name}", ids) end end end diff --git a/app/workers/scheduler/instance_refresh_scheduler.rb b/app/workers/scheduler/instance_refresh_scheduler.rb index 59f91dc62..818682a5d 100644 --- a/app/workers/scheduler/instance_refresh_scheduler.rb +++ b/app/workers/scheduler/instance_refresh_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::InstanceRefreshScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform Instance.refresh diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb index 8f607db03..f78c0584d 100644 --- a/app/workers/scheduler/ip_cleanup_scheduler.rb +++ b/app/workers/scheduler/ip_cleanup_scheduler.rb @@ -6,7 +6,7 @@ class Scheduler::IpCleanupScheduler IP_RETENTION_PERIOD = ENV.fetch('IP_RETENTION_PERIOD', 1.year).to_i.seconds.freeze SESSION_RETENTION_PERIOD = ENV.fetch('SESSION_RETENTION_PERIOD', 1.year).to_i.seconds.freeze - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform clean_ip_columns! diff --git a/app/workers/scheduler/pghero_scheduler.rb b/app/workers/scheduler/pghero_scheduler.rb index a756b13b9..49bf2db7a 100644 --- a/app/workers/scheduler/pghero_scheduler.rb +++ b/app/workers/scheduler/pghero_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::PgheroScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform PgHero.capture_space_stats diff --git a/app/workers/scheduler/scheduled_statuses_scheduler.rb b/app/workers/scheduler/scheduled_statuses_scheduler.rb index 3bf6300b3..b5801248f 100644 --- a/app/workers/scheduler/scheduled_statuses_scheduler.rb +++ b/app/workers/scheduler/scheduled_statuses_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::ScheduledStatusesScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform publish_scheduled_statuses! diff --git a/app/workers/scheduler/suspended_user_cleanup_scheduler.rb b/app/workers/scheduler/suspended_user_cleanup_scheduler.rb index 87e22161b..90feead67 100644 --- a/app/workers/scheduler/suspended_user_cleanup_scheduler.rb +++ b/app/workers/scheduler/suspended_user_cleanup_scheduler.rb @@ -16,7 +16,7 @@ class Scheduler::SuspendedUserCleanupScheduler # has the capacity for it. MAX_DELETIONS_PER_JOB = 10 - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform return if Sidekiq::Queue.new('pull').size > MAX_PULL_SIZE diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb index 4aee7935a..2d2efc731 100644 --- a/app/workers/scheduler/user_cleanup_scheduler.rb +++ b/app/workers/scheduler/user_cleanup_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::UserCleanupScheduler include Sidekiq::Worker - sidekiq_options retry: 0 + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform clean_unconfirmed_accounts! diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb index 9c040f6e4..1c9a2aabe 100644 --- a/app/workers/scheduler/vacuum_scheduler.rb +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -3,7 +3,7 @@ class Scheduler::VacuumScheduler include Sidekiq::Worker - sidekiq_options retry: 0, lock: :until_executed + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform vacuum_operations.each do |operation| diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 2278329a5..12c45c22a 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -23,7 +23,7 @@ class: Scheduler::Trends::ReviewNotificationsScheduler queue: scheduler indexing_scheduler: - every: '5m' + interval: 1 minute class: Scheduler::IndexingScheduler queue: scheduler vacuum_scheduler: From 6b896b20cc87f9665ded7ed36d1ace32438ca9e3 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 3 Aug 2023 11:12:52 +0200 Subject: [PATCH 4/5] Add primary key to preview_cards_statuses join table (includes deduplication migration) (#25243) --- app/services/fetch_link_card_service.rb | 10 +++-- ..._unique_index_on_preview_cards_statuses.rb | 39 +++++++++++++++++++ ...ey_to_preview_cards_statuses_join_table.rb | 20 ++++++++++ db/schema.rb | 4 +- lib/tasks/tests.rake | 25 ++++++++++++ 5 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb create mode 100644 db/post_migrate/20230803112520_add_primary_key_to_preview_cards_statuses_join_table.rb diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 886582460..13775e63c 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -61,9 +61,13 @@ class FetchLinkCardService < BaseService end def attach_card - @status.preview_cards << @card - Rails.cache.delete(@status) - Trends.links.register(@status) + with_redis_lock("attach_card:#{@status.id}") do + return if @status.preview_cards.any? + + @status.preview_cards << @card + Rails.cache.delete(@status) + Trends.links.register(@status) + end end def parse_urls diff --git a/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb b/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb new file mode 100644 index 000000000..936d7840e --- /dev/null +++ b/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class AddUniqueIndexOnPreviewCardsStatuses < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def up + add_index :preview_cards_statuses, [:status_id, :preview_card_id], name: :preview_cards_statuses_pkey, algorithm: :concurrently, unique: true + rescue ActiveRecord::RecordNotUnique + deduplicate_and_reindex! + end + + def down + remove_index :preview_cards_statuses, name: :preview_cards_statuses_pkey + end + + private + + def deduplicate_and_reindex! + deduplicate_preview_cards! + + safety_assured { execute 'REINDEX INDEX preview_cards_statuses_pkey' } + rescue ActiveRecord::RecordNotUnique + retry + end + + def deduplicate_preview_cards! + # Statuses should have only one preview card at most, even if that's not the database + # constraint we will end up with + duplicate_ids = select_all('SELECT status_id FROM preview_cards_statuses GROUP BY status_id HAVING count(*) > 1;').rows + + duplicate_ids.each_slice(1000) do |ids| + # This one is tricky: since we don't have primary keys to keep only one record, + # use the physical `ctid` + safety_assured do + execute "DELETE FROM preview_cards_statuses p WHERE p.status_id IN (#{ids.join(', ')}) AND p.ctid NOT IN (SELECT q.ctid FROM preview_cards_statuses q WHERE q.status_id = p.status_id LIMIT 1)" + end + end + end +end diff --git a/db/post_migrate/20230803112520_add_primary_key_to_preview_cards_statuses_join_table.rb b/db/post_migrate/20230803112520_add_primary_key_to_preview_cards_statuses_join_table.rb new file mode 100644 index 000000000..34877ac67 --- /dev/null +++ b/db/post_migrate/20230803112520_add_primary_key_to_preview_cards_statuses_join_table.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddPrimaryKeyToPreviewCardsStatusesJoinTable < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def up + safety_assured do + execute 'ALTER TABLE preview_cards_statuses ADD PRIMARY KEY USING INDEX preview_cards_statuses_pkey' + end + end + + def down + safety_assured do + # I have found no way to demote the primary key to an index, instead, re-create the index + execute 'CREATE UNIQUE INDEX CONCURRENTLY preview_cards_statuses_pkey_tmp ON preview_cards_statuses (status_id, preview_card_id)' + execute 'ALTER TABLE preview_cards_statuses DROP CONSTRAINT preview_cards_statuses_pkey' + execute 'ALTER INDEX preview_cards_statuses_pkey_tmp RENAME TO preview_cards_statuses_pkey' + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 59ab9dbed..0c8ede383 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do +ActiveRecord::Schema[7.0].define(version: 2023_08_03_112520) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -805,7 +805,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do t.index ["url"], name: "index_preview_cards_on_url", unique: true end - create_table "preview_cards_statuses", id: false, force: :cascade do |t| + create_table "preview_cards_statuses", primary_key: ["status_id", "preview_card_id"], force: :cascade do |t| t.bigint "preview_card_id", null: false t.bigint "status_id", null: false t.index ["status_id", "preview_card_id"], name: "index_preview_cards_statuses_on_status_id_and_preview_card_id" diff --git a/lib/tasks/tests.rake b/lib/tasks/tests.rake index 3c88ce450..ef4d46fe0 100644 --- a/lib/tasks/tests.rake +++ b/lib/tasks/tests.rake @@ -63,6 +63,11 @@ namespace :tests do puts 'Account domains not properly normalized' exit(1) end + + unless Status.find(12).preview_cards.pluck(:url) == ['https://joinmastodon.org/'] + puts 'Preview cards not deduplicated as expected' + exit(1) + end end desc 'Populate the database with test data for 2.4.3' @@ -238,6 +243,11 @@ namespace :tests do (10, 2, '@admin hey!', NULL, 1, 3, now(), now()), (11, 1, '@user hey!', 10, 1, 3, now(), now()); + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (12, 1, 'check out https://joinmastodon.org/', now(), now()); + -- mentions (from previous statuses) INSERT INTO "mentions" @@ -326,6 +336,21 @@ namespace :tests do (1, 6, 2, 'Follow', 2, now(), now()), (2, 2, 1, 'Mention', 4, now(), now()), (3, 1, 2, 'Mention', 5, now(), now()); + + -- preview cards + + INSERT INTO "preview_cards" + (id, url, title, created_at, updated_at) + VALUES + (1, 'https://joinmastodon.org/', 'Mastodon - Decentralized social media', now(), now()); + + -- many-to-many association between preview cards and statuses + + INSERT INTO "preview_cards_statuses" + (status_id, preview_card_id) + VALUES + (12, 1), + (12, 1); SQL end end From 3a4d3e9d4b573c400eec1743471d54cdccae50a5 Mon Sep 17 00:00:00 2001 From: Daniel M Brasil Date: Thu, 3 Aug 2023 06:25:47 -0300 Subject: [PATCH 5/5] Add `GET /api/v1/instance/languages` to REST API (#24443) Co-authored-by: Eugen Rochko Co-authored-by: Claire --- .../api/v1/instances/languages_controller.rb | 21 +++++++++++++++++++ app/presenters/language_presenter.rb | 20 ++++++++++++++++++ app/serializers/rest/language_serializer.rb | 5 +++++ config/routes/api.rb | 1 + .../api/v1/instances/languages_spec.rb | 19 +++++++++++++++++ 5 files changed, 66 insertions(+) create mode 100644 app/controllers/api/v1/instances/languages_controller.rb create mode 100644 app/presenters/language_presenter.rb create mode 100644 app/serializers/rest/language_serializer.rb create mode 100644 spec/requests/api/v1/instances/languages_spec.rb diff --git a/app/controllers/api/v1/instances/languages_controller.rb b/app/controllers/api/v1/instances/languages_controller.rb new file mode 100644 index 000000000..17509e748 --- /dev/null +++ b/app/controllers/api/v1/instances/languages_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::Instances::LanguagesController < Api::BaseController + skip_before_action :require_authenticated_user!, unless: :limited_federation_mode? + skip_around_action :set_locale + + before_action :set_languages + + vary_by '' + + def show + cache_even_if_authenticated! + render json: @languages, each_serializer: REST::LanguageSerializer + end + + private + + def set_languages + @languages = LanguagesHelper::SUPPORTED_LOCALES.keys.map { |code| LanguagePresenter.new(code) } + end +end diff --git a/app/presenters/language_presenter.rb b/app/presenters/language_presenter.rb new file mode 100644 index 000000000..69ea991d5 --- /dev/null +++ b/app/presenters/language_presenter.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class LanguagePresenter < ActiveModelSerializers::Model + attributes :code, :name, :native_name + + def initialize(code) + super() + + @code = code + @item = LanguagesHelper::SUPPORTED_LOCALES[code] + end + + def name + @item[0] + end + + def native_name + @item[1] + end +end diff --git a/app/serializers/rest/language_serializer.rb b/app/serializers/rest/language_serializer.rb new file mode 100644 index 000000000..e07956465 --- /dev/null +++ b/app/serializers/rest/language_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class REST::LanguageSerializer < ActiveModel::Serializer + attributes :code, :name +end diff --git a/config/routes/api.rb b/config/routes/api.rb index 6db03147b..48c4c6522 100644 --- a/config/routes/api.rb +++ b/config/routes/api.rb @@ -120,6 +120,7 @@ namespace :api, format: false do resource :privacy_policy, only: [:show], controller: 'instances/privacy_policies' resource :extended_description, only: [:show], controller: 'instances/extended_descriptions' resource :translation_languages, only: [:show], controller: 'instances/translation_languages' + resource :languages, only: [:show], controller: 'instances/languages' resource :activity, only: [:show], controller: 'instances/activity' end diff --git a/spec/requests/api/v1/instances/languages_spec.rb b/spec/requests/api/v1/instances/languages_spec.rb new file mode 100644 index 000000000..8ab8bf99c --- /dev/null +++ b/spec/requests/api/v1/instances/languages_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'Languages' do + describe 'GET /api/v1/instance/languages' do + before do + get '/api/v1/instance/languages' + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns the supported languages' do + expect(body_as_json.pluck(:code)).to match_array LanguagesHelper::SUPPORTED_LOCALES.keys.map(&:to_s) + end + end +end