From d627c4c261c5888a967b2c0e20aa879c4bd6089e Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 4 Jul 2020 20:19:38 +0200 Subject: [PATCH 1/2] Update rack --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bbafb0ad..08c4db47 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -136,7 +136,7 @@ GEM coffee-script-source (1.12.2) colorize (0.8.1) concurrent-ruby (1.1.6) - connection_pool (2.2.2) + connection_pool (2.2.3) crass (1.0.6) database_cleaner (1.8.5) debug_inspector (0.0.3) @@ -351,7 +351,7 @@ GEM public_suffix (4.0.5) puma (4.3.5) nio4r (~> 2.0) - rack (2.0.9) + rack (2.2.3) rack-pjax (1.1.0) nokogiri (~> 1.5) rack (>= 1.1) @@ -479,11 +479,11 @@ GEM sprockets-rails tilt shellany (0.0.1) - sidekiq (5.2.8) + sidekiq (5.2.9) connection_pool (~> 2.2, >= 2.2.2) - rack (< 2.1.0) + rack (~> 2.0) rack-protection (>= 1.5.0) - redis (>= 3.3.5, < 5) + redis (>= 3.3.5, < 4.2) simple_oauth (0.3.1) simplecov (0.18.5) docile (~> 1.1) From efec568d435c3e0ff53b54e67b7968a0a7660cb4 Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sat, 4 Jul 2020 20:12:53 +0200 Subject: [PATCH 2/2] Use timestamp-based IDs This change introduces timestamp-based (Snowflake/Cornflake/Waspflake) identifier. --- Rakefile | 29 +++++++++ .../20200704163504_use_timestamped_ids.rb | 54 ++++++++++++++++ db/schema.rb | 64 +++++++++---------- 3 files changed, 115 insertions(+), 32 deletions(-) create mode 100644 db/migrate/20200704163504_use_timestamped_ids.rb diff --git a/Rakefile b/Rakefile index 8e99145c..3868ad1e 100644 --- a/Rakefile +++ b/Rakefile @@ -430,3 +430,32 @@ namespace :justask do end end end + +namespace :db do + namespace :schema do + task :create_timestampid_function do + conn = ActiveRecord::Base.connection + have_func = conn.execute("SELECT EXISTS(SELECT * FROM pg_proc WHERE proname = 'gen_timestamp_id');").values.first.first + next if have_func + + statement = File.read(File.join(__dir__, 'db/migrate/20200704163504_use_timestamped_ids.rb')).match(/<~SQL(?.+)SQL$/m)[:stmt] + conn.execute(statement) + end + + task :create_id_sequences do + conn = ActiveRecord::Base.connection + + # required for the timestampid function to work properly + %i[questions answers comments smiles comment_smiles users].each do |tbl| + conn.execute("CREATE SEQUENCE IF NOT EXISTS #{tbl}_id_seq") + end + end + end + + # create timestampid before loading schema + Rake::Task['db:schema:load'].enhance ['db:schema:create_timestampid_function'] + # create id_sequences after loading schema + Rake::Task['db:schema:load'].enhance do + Rake::Task['db:schema:create_id_sequences'].invoke + end +end diff --git a/db/migrate/20200704163504_use_timestamped_ids.rb b/db/migrate/20200704163504_use_timestamped_ids.rb new file mode 100644 index 00000000..49b44bd7 --- /dev/null +++ b/db/migrate/20200704163504_use_timestamped_ids.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'securerandom' + +# This migration changes the IDs of several tables from serial to a +# timestamped/"snowflake" one. +# +# Instead of "snowflakes" we shall call those IDs Cornflakes or Waspflakes +# instead. +# +# Implementation somewhat lifted from Mastodon. +class UseTimestampedIds < ActiveRecord::Migration[5.2] + def up + # PL/pgSQL is just spicy pascal + # don't @ me + execute <<~SQL + CREATE or replace FUNCTION gen_timestamp_id(tblname text) RETURNS bigint AS $$ + DECLARE + timepart bigint; + seqpart bigint; + BEGIN + timepart := (date_part('epoch', now()) * 1000)::bigint << 16; + seqpart := ('x' || substr(md5(tblname || + '#{SecureRandom.hex(16)}' || + timepart::text), 1, 4))::bit(16)::bigint; + RETURN timepart | ((seqpart + nextval(tblname || '_id_seq')) & 65535); + END; + $$ LANGUAGE plpgsql VOLATILE; + SQL + + # we need to migrate related columns to bigints for this to work + { + question: %i[answers inboxes], + answer: %i[comments smiles subscriptions], + comment: %i[comment_smiles], + user: %i[announcements answers comment_smiles comments inboxes list_members lists moderation_comments moderation_votes questions reports services smiles subscriptions themes users_roles], + + # polymorphic tables go brrr + recipient: %i[notifications], + source: %i[relationships], + target: %i[notifications relationships reports], + }.each do |ref, tbls| + tbls.each do |tbl| + say "Migrating #{tbl}.#{ref}_id to bigint" + change_column(tbl, :"#{ref}_id", :bigint) + end + end + + %i[questions answers comments smiles comment_smiles users].each do |tbl| + say "Migrating #{tbl} to use timestamped ids" + change_column(tbl, :id, :bigint, default: -> { "gen_timestamp_id('#{tbl}'::text)" }) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ae76782d..f399bfce 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_05_25_145144) do +ActiveRecord::Schema.define(version: 2020_07_04_163504) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -27,11 +27,11 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do t.index ["user_id"], name: "index_announcements_on_user_id" end - create_table "answers", id: :serial, force: :cascade do |t| + create_table "answers", id: :bigint, default: -> { "gen_timestamp_id('answers'::text)" }, force: :cascade do |t| t.text "content" - t.integer "question_id" + t.bigint "question_id" t.integer "comment_count", default: 0, null: false - t.integer "user_id" + t.bigint "user_id" t.datetime "created_at" t.datetime "updated_at" t.integer "smile_count", default: 0, null: false @@ -39,9 +39,9 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do t.index ["user_id", "created_at"], name: "index_answers_on_user_id_and_created_at" end - create_table "comment_smiles", id: :serial, force: :cascade do |t| - t.integer "user_id" - t.integer "comment_id" + create_table "comment_smiles", id: :bigint, default: -> { "gen_timestamp_id('comment_smiles'::text)" }, force: :cascade do |t| + t.bigint "user_id" + t.bigint "comment_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["comment_id"], name: "index_comment_smiles_on_comment_id" @@ -49,10 +49,10 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do t.index ["user_id"], name: "index_comment_smiles_on_user_id" end - create_table "comments", id: :serial, force: :cascade do |t| + create_table "comments", id: :bigint, default: -> { "gen_timestamp_id('comments'::text)" }, force: :cascade do |t| t.string "content" - t.integer "answer_id" - t.integer "user_id" + t.bigint "answer_id" + t.bigint "user_id" t.datetime "created_at" t.datetime "updated_at" t.integer "smile_count", default: 0, null: false @@ -61,8 +61,8 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do end create_table "inboxes", id: :serial, force: :cascade do |t| - t.integer "user_id" - t.integer "question_id" + t.bigint "user_id" + t.bigint "question_id" t.boolean "new" t.datetime "created_at" t.datetime "updated_at" @@ -71,14 +71,14 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do create_table "list_members", id: :serial, force: :cascade do |t| t.integer "list_id", null: false - t.integer "user_id", null: false + t.bigint "user_id", null: false t.datetime "created_at" t.datetime "updated_at" t.index ["list_id", "user_id"], name: "index_list_members_on_list_id_and_user_id", unique: true end create_table "lists", id: :serial, force: :cascade do |t| - t.integer "user_id", null: false + t.bigint "user_id", null: false t.string "name" t.string "display_name" t.boolean "private", default: true @@ -89,7 +89,7 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do create_table "moderation_comments", id: :serial, force: :cascade do |t| t.integer "report_id" - t.integer "user_id" + t.bigint "user_id" t.string "content" t.datetime "created_at" t.datetime "updated_at" @@ -98,7 +98,7 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do create_table "moderation_votes", id: :serial, force: :cascade do |t| t.integer "report_id", null: false - t.integer "user_id", null: false + t.bigint "user_id", null: false t.boolean "upvote", default: false, null: false t.datetime "created_at" t.datetime "updated_at" @@ -109,19 +109,19 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do create_table "notifications", id: :serial, force: :cascade do |t| t.string "target_type" - t.integer "target_id" - t.integer "recipient_id" + t.bigint "target_id" + t.bigint "recipient_id" t.boolean "new" t.datetime "created_at" t.datetime "updated_at" end - create_table "questions", id: :serial, force: :cascade do |t| + create_table "questions", id: :bigint, default: -> { "gen_timestamp_id('questions'::text)" }, force: :cascade do |t| t.string "content" t.boolean "author_is_anonymous" t.string "author_name" t.string "author_email" - t.integer "user_id" + t.bigint "user_id" t.datetime "created_at" t.datetime "updated_at" t.integer "answer_count", default: 0, null: false @@ -129,8 +129,8 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do end create_table "relationships", id: :serial, force: :cascade do |t| - t.integer "source_id" - t.integer "target_id" + t.bigint "source_id" + t.bigint "target_id" t.datetime "created_at" t.datetime "updated_at" t.index ["source_id", "target_id"], name: "index_relationships_on_source_id_and_target_id", unique: true @@ -140,8 +140,8 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do create_table "reports", id: :serial, force: :cascade do |t| t.string "type", null: false - t.integer "target_id", null: false - t.integer "user_id", null: false + t.bigint "target_id", null: false + t.bigint "user_id", null: false t.datetime "created_at" t.datetime "updated_at" t.boolean "deleted", default: false @@ -162,7 +162,7 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do create_table "services", id: :serial, force: :cascade do |t| t.string "type", null: false - t.integer "user_id", null: false + t.bigint "user_id", null: false t.string "uid" t.string "access_token" t.string "access_secret" @@ -172,9 +172,9 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do t.index ["user_id"], name: "index_services_on_user_id" end - create_table "smiles", id: :serial, force: :cascade do |t| - t.integer "user_id" - t.integer "answer_id" + create_table "smiles", id: :bigint, default: -> { "gen_timestamp_id('smiles'::text)" }, force: :cascade do |t| + t.bigint "user_id" + t.bigint "answer_id" t.datetime "created_at" t.datetime "updated_at" t.index ["answer_id"], name: "index_smiles_on_answer_id" @@ -183,8 +183,8 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do end create_table "subscriptions", id: :serial, force: :cascade do |t| - t.integer "user_id", null: false - t.integer "answer_id", null: false + t.bigint "user_id", null: false + t.bigint "answer_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.boolean "is_active", default: true @@ -192,7 +192,7 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do end create_table "themes", id: :serial, force: :cascade do |t| - t.integer "user_id", null: false + t.bigint "user_id", null: false t.integer "primary_color", default: 6174129 t.integer "primary_text", default: 16777215 t.integer "danger_color", default: 14431557 @@ -219,7 +219,7 @@ ActiveRecord::Schema.define(version: 2020_05_25_145144) do t.index ["user_id", "created_at"], name: "index_themes_on_user_id_and_created_at" end - create_table "users", id: :serial, force: :cascade do |t| + create_table "users", id: :bigint, default: -> { "gen_timestamp_id('users'::text)" }, force: :cascade do |t| t.string "email", default: "", null: false t.string "encrypted_password", default: "", null: false t.string "reset_password_token"