#222 [perf] db indexes & usernameNorm

migrations/030-indexes.sql

@@ -0,0 +1,54 @@
+-- Up
+
+ALTER TABLE users ADD COLUMN usernameNorm TEXT NULL; -- shouldn't be null, but we first need to populate it from JS
+CREATE UNIQUE INDEX "users_usernameNorm" ON "users" ("usernameNorm");
+
+CREATE INDEX "authenticators_type" ON "authenticators" ("type");
+CREATE INDEX "authenticators_userId" ON "authenticators" ("userId");
+
+CREATE INDEX "census_locale_edition" ON "census" ("locale", "edition");
+CREATE INDEX "census_fingerprint" ON "census" ("fingerprint");
+
+CREATE INDEX "emails_email" ON "emails" ("email");
+
+CREATE INDEX "nouns_locale" ON "nouns" ("locale");
+CREATE INDEX "nouns_masc" ON "nouns" ("masc");
+
+CREATE INDEX "inclusive_locale" ON "inclusive" ("locale");
+CREATE INDEX "inclusive_insteadOf" ON "inclusive" ("insteadOf");
+
+CREATE INDEX "terms_locale" ON "terms" ("locale");
+CREATE INDEX "terms_term" ON "terms" ("term");
+
+CREATE INDEX "profiles_locale" ON "profiles" ("locale");
+CREATE INDEX "profiles_userId" ON "profiles" ("userId");
+CREATE INDEX "profiles_locale_userId" ON "profiles" ("locale", "userId");
+
+CREATE INDEX "sources_locale" ON "sources" ("locale");
+
+-- Down
+
+DROP INDEX "users_usernameNorm";
+
+DROP INDEX "authenticators_type";
+DROP INDEX "authenticators_userId";
+
+DROP INDEX "census_locale_edition";
+DROP INDEX "census_fingerprint";
+
+DROP INDEX "emails_email";
+
+DROP INDEX "nouns_locale";
+DROP INDEX "nouns_masc";
+
+DROP INDEX "inclusive_locale";
+DROP INDEX "inclusive_insteadOf";
+
+DROP INDEX "terms_locale";
+DROP INDEX "terms_term";
+
+DROP INDEX "profiles_locale";
+DROP INDEX "profiles_userId";
+DROP INDEX "profiles_locale_userId";
+
+DROP INDEX "sources_locale";

server/migrate.js

@@ -7,3 +7,17 @@ async function migrate() {
 }
 
 migrate();
+
+// temporary code for a non-sqlite migration:
+const normalise = s => s.trim().toLowerCase();
+
+async function populateUsernameNorm() {
+    const db = await dbConnection();
+    for ({id, username} of await db.all(`SELECT id, username FROM users WHERE usernameNorm IS NULL`)) {
+        const norm = normalise(username); // username is safe, so so will be norm
+        console.log(username, norm)
+        await db.all(`UPDATE users SET usernameNorm = '${norm}' WHERE id = '${id}'`);
+    }
+    await db.close();
+}
+populateUsernameNorm();

server/routes/admin.js

@@ -133,7 +133,7 @@ router.post('/admin/ban/:username', handleErrorAsync(async (req, res) => {
     await req.db.get(SQL`
         UPDATE users
         SET bannedReason = ${req.body.reason || null} 
-        WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(req.params.username)}
+        WHERE usernameNorm = ${normalise(req.params.username)}
     `);
 
     return res.json(true);

server/routes/profile.js

@@ -28,8 +28,7 @@ const calcAge = birthday => {
 const fetchProfiles = async (db, username, self, isAdmin) => {
     const profiles = await db.all(SQL`
         SELECT profiles.*, users.id, users.username, users.email, users.avatarSource, users.bannedReason, users.roles FROM profiles LEFT JOIN users on users.id == profiles.userId 
-        WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(username)}
+        WHERE usernameNorm = ${normalise(username)}
-        AND profiles.active = 1
         ORDER BY profiles.locale
     `);
 

server/routes/user.js

@@ -73,7 +73,7 @@ const defaultUsername = async (db, email) => {
     let c = 0;
     while (true) {
         let proposal = base + (c || '');
-        let dbUser = await db.get(SQL`SELECT id FROM users WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(proposal)}`);
+        let dbUser = await db.get(SQL`SELECT id FROM users WHERE usernameNorm = ${normalise(proposal)}`);
         if (!dbUser) {
             return proposal;
         }
@@ -91,8 +91,8 @@ const fetchOrCreateUser = async (db, user, avatarSource = 'gravatar') => {
             roles: '',
             avatarSource: avatarSource,
         }
-        await db.get(SQL`INSERT INTO users(id, username, email, roles, avatarSource)
+        await db.get(SQL`INSERT INTO users(id, username, usernameNorm, email, roles, avatarSource)
-            VALUES (${dbUser.id}, ${dbUser.username}, ${dbUser.email}, ${dbUser.roles}, ${dbUser.avatarSource})`)
+            VALUES (${dbUser.id}, ${dbUser.username}, ${normalise(dbUser.username)}, ${dbUser.email}, ${dbUser.roles}, ${dbUser.avatarSource})`)
     }
 
     dbUser.avatar = await avatar(db, dbUser);
@@ -195,7 +195,7 @@ router.post('/user/init', handleErrorAsync(async (req, res) => {
     if (isEmail) {
         user = await req.db.get(SQL`SELECT * FROM users WHERE email = ${normalise(usernameOrEmail)}`);
     } else {
-        user = await req.db.get(SQL`SELECT * FROM users WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(usernameOrEmail)}`);
+        user = await req.db.get(SQL`SELECT * FROM users WHERE usernameNorm = ${normalise(usernameOrEmail)}`);
     }
 
     if (!user && !isEmail) {
@@ -268,12 +268,12 @@ router.post('/user/change-username', handleErrorAsync(async (req, res) => {
         return res.json({ error: 'user.account.changeUsername.invalid' });
     }
 
-    const dbUser = await req.db.get(SQL`SELECT * FROM users WHERE lower(trim(replace(replace(replace(replace(replace(replace(replace(replace(replace(username, 'Ą', 'ą'), 'Ć', 'ć'), 'Ę', 'ę'), 'Ł', 'ł'), 'Ń', 'ń'), 'Ó', 'ó'), 'Ś', 'ś'), 'Ż', 'ż'), 'Ź', 'ż'))) = ${normalise(req.body.username)}`);
+    const dbUser = await req.db.get(SQL`SELECT * FROM users WHERE usernameNorm = ${normalise(req.body.username)}`);
     if (dbUser) {
         return res.json({ error: 'user.account.changeUsername.taken' })
     }
 
-    await req.db.get(SQL`UPDATE users SET username = ${req.body.username} WHERE id = ${req.user.id}`);
+    await req.db.get(SQL`UPDATE users SET username = ${req.body.username}, usernameNorm = ${normalise(req.body.username)} WHERE id = ${req.user.id}`);
 
     return res.json({token: await issueAuthentication(req.db, req.user)});
 }));