From 8d8ffac322d7997c6b8f1eafa3a9e394618edeb5 Mon Sep 17 00:00:00 2001 From: Avris Date: Wed, 14 Jul 2021 15:28:53 +0200 Subject: [PATCH] #222 [perf] db indexes & usernameNorm --- migrations/030-indexes.sql | 54 ++++++++++++++++++++++++++++++++++++++ server/migrate.js | 14 ++++++++++ server/routes/admin.js | 2 +- server/routes/profile.js | 3 +-- server/routes/user.js | 12 ++++----- 5 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 migrations/030-indexes.sql diff --git a/migrations/030-indexes.sql b/migrations/030-indexes.sql new file mode 100644 index 00000000..4d424e62 --- /dev/null +++ b/migrations/030-indexes.sql @@ -0,0 +1,54 @@ +-- Up + +ALTER TABLE users ADD COLUMN usernameNorm TEXT NULL; -- shouldn't be null, but we first need to populate it from JS +CREATE UNIQUE INDEX "users_usernameNorm" ON "users" ("usernameNorm"); + +CREATE INDEX "authenticators_type" ON "authenticators" ("type"); +CREATE INDEX "authenticators_userId" ON "authenticators" ("userId"); + +CREATE INDEX "census_locale_edition" ON "census" ("locale", "edition"); +CREATE INDEX "census_fingerprint" ON "census" ("fingerprint"); + +CREATE INDEX "emails_email" ON "emails" ("email"); + +CREATE INDEX "nouns_locale" ON "nouns" ("locale"); +CREATE INDEX "nouns_masc" ON "nouns" ("masc"); + +CREATE INDEX "inclusive_locale" ON "inclusive" ("locale"); +CREATE INDEX "inclusive_insteadOf" ON "inclusive" ("insteadOf"); + +CREATE INDEX "terms_locale" ON "terms" ("locale"); +CREATE INDEX "terms_term" ON "terms" ("term"); + +CREATE INDEX "profiles_locale" ON "profiles" ("locale"); +CREATE INDEX "profiles_userId" ON "profiles" ("userId"); +CREATE INDEX "profiles_locale_userId" ON "profiles" ("locale", "userId"); + +CREATE INDEX "sources_locale" ON "sources" ("locale"); + +-- Down + +DROP INDEX "users_usernameNorm"; + +DROP INDEX "authenticators_type"; +DROP INDEX "authenticators_userId"; + +DROP INDEX "census_locale_edition"; +DROP INDEX "census_fingerprint"; + +DROP INDEX "emails_email"; + +DROP INDEX "nouns_locale"; +DROP INDEX "nouns_masc"; + +DROP INDEX "inclusive_locale"; +DROP INDEX "inclusive_insteadOf"; + +DROP INDEX "terms_locale"; +DROP INDEX "terms_term"; + +DROP INDEX "profiles_locale"; +DROP INDEX "profiles_userId"; +DROP INDEX "profiles_locale_userId"; + +DROP INDEX "sources_locale"; diff --git a/server/migrate.js b/server/migrate.js index 02319de4..85895106 100644 --- a/server/migrate.js +++ b/server/migrate.js @@ -7,3 +7,17 @@ async function migrate() { } migrate(); + +// temporary code for a non-sqlite migration: +const normalise = s => s.trim().toLowerCase(); + +async function populateUsernameNorm() { + const db = await dbConnection(); + for ({id, username} of await db.all(`SELECT id, username FROM users WHERE usernameNorm IS NULL`)) { + const norm = normalise(username); // username is safe, so so will be norm + console.log(username, norm) + await db.all(`UPDATE users SET usernameNorm = '${norm}' WHERE id = '${id}'`); + } + await db.close(); +} +populateUsernameNorm(); diff --git a/server/routes/admin.js b/server/routes/admin.js index da0ee4a4..2626d97f 100644 --- a/server/routes/admin.js +++ b/server/routes/admin.js @@ -133,7 +133,7 @@ router.post('/admin/ban/:username', handleErrorAsync(async (req, res) => { await req.db.get(SQL` UPDATE users SET bannedReason = ${req.body.reason || null} - WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(req.params.username)} + WHERE usernameNorm = ${normalise(req.params.username)} `); return res.json(true); diff --git a/server/routes/profile.js b/server/routes/profile.js index 3a4f3d1d..c460f8e8 100644 --- a/server/routes/profile.js +++ b/server/routes/profile.js @@ -28,8 +28,7 @@ const calcAge = birthday => { const fetchProfiles = async (db, username, self, isAdmin) => { const profiles = await db.all(SQL` SELECT profiles.*, users.id, users.username, users.email, users.avatarSource, users.bannedReason, users.roles FROM profiles LEFT JOIN users on users.id == profiles.userId - WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(username)} - AND profiles.active = 1 + WHERE usernameNorm = ${normalise(username)} ORDER BY profiles.locale `); diff --git a/server/routes/user.js b/server/routes/user.js index 2c388549..4d1c8d24 100644 --- a/server/routes/user.js +++ b/server/routes/user.js @@ -73,7 +73,7 @@ const defaultUsername = async (db, email) => { let c = 0; while (true) { let proposal = base + (c || ''); - let dbUser = await db.get(SQL`SELECT id FROM users WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(proposal)}`); + let dbUser = await db.get(SQL`SELECT id FROM users WHERE usernameNorm = ${normalise(proposal)}`); if (!dbUser) { return proposal; } @@ -91,8 +91,8 @@ const fetchOrCreateUser = async (db, user, avatarSource = 'gravatar') => { roles: '', avatarSource: avatarSource, } - await db.get(SQL`INSERT INTO users(id, username, email, roles, avatarSource) - VALUES (${dbUser.id}, ${dbUser.username}, ${dbUser.email}, ${dbUser.roles}, ${dbUser.avatarSource})`) + await db.get(SQL`INSERT INTO users(id, username, usernameNorm, email, roles, avatarSource) + VALUES (${dbUser.id}, ${dbUser.username}, ${normalise(dbUser.username)}, ${dbUser.email}, ${dbUser.roles}, ${dbUser.avatarSource})`) } dbUser.avatar = await avatar(db, dbUser); @@ -195,7 +195,7 @@ router.post('/user/init', handleErrorAsync(async (req, res) => { if (isEmail) { user = await req.db.get(SQL`SELECT * FROM users WHERE email = ${normalise(usernameOrEmail)}`); } else { - user = await req.db.get(SQL`SELECT * FROM users WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(usernameOrEmail)}`); + user = await req.db.get(SQL`SELECT * FROM users WHERE usernameNorm = ${normalise(usernameOrEmail)}`); } if (!user && !isEmail) { @@ -268,12 +268,12 @@ router.post('/user/change-username', handleErrorAsync(async (req, res) => { return res.json({ error: 'user.account.changeUsername.invalid' }); } - const dbUser = await req.db.get(SQL`SELECT * FROM users WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(req.body.username)}`); + const dbUser = await req.db.get(SQL`SELECT * FROM users WHERE usernameNorm = ${normalise(req.body.username)}`); if (dbUser) { return res.json({ error: 'user.account.changeUsername.taken' }) } - await req.db.get(SQL`UPDATE users SET username = ${req.body.username} WHERE id = ${req.user.id}`); + await req.db.get(SQL`UPDATE users SET username = ${req.body.username}, usernameNorm = ${normalise(req.body.username)} WHERE id = ${req.user.id}`); return res.json({token: await issueAuthentication(req.db, req.user)}); }));