KayFaraday
/
Zaimki
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

#216 optimise stats

This commit is contained in:
Avris 2021-06-10 08:45:13 +02:00
parent 0eeafc3f3a
commit 4deba3034c
12 changed files with 172 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.dist
View File

@ -2,6 +2,8 @@ BASE_URL=http://localhost:3000
SECRET=
STATS_FILE=%projectdir%/stats.json
MAILER_HOST=
MAILER_PORT=
MAILER_USER=

2
.gitignore vendored
View File

@ -8,6 +8,8 @@
/keys
/stats.json
# Created by .ignore support plugin (hsz.mobi)
### Node template
# Logs

2
layouts/default.vue
View File

@ -40,7 +40,7 @@
reject();
});
});
}
};
}
}
</script>

2
locale/pl/config.suml
View File

@ -194,7 +194,7 @@ links:
icon: 'play-circle'
url: 'https://www.facebook.com/gdynaukajestkobieta/videos/193680249317891'
headline: 'Tip-top. Krok w stronę feminatywów.'
extra: 'Sybil, (Gdy Nauka jest Kobietą UAM)'
extra: 'Sybil (Gdy Nauka jest Kobietą UAM)'
links:
-
icon: 'globe-europe'

1
nuxt.config.js
View File

@ -153,6 +153,7 @@ export default {
LOCALES: locales,
FLAGS: buildFlags(),
BUCKET: `https://${process.env.AWS_S3_BUCKET}.s3-${process.env.AWS_REGION}.amazonaws.com`,
STATS_FILE: process.env.STATS_FILE,
},
serverMiddleware: ['~/server/index.js'],
axios: {

6
plugins/globals.js
View File

@ -2,6 +2,7 @@ import Vue from 'vue'
import t from '../src/translator';
import config from '../data/config.suml';
import {buildDict} from "../src/helpers";
import {DateTime} from "luxon";
export default ({ app, store }) => {
Vue.prototype.$eventHub = new Vue();
@ -41,4 +42,9 @@ export default ({ app, store }) => {
s.classList.add(`${name}-script`);
document.body.appendChild(s);
};
Vue.prototype.$datetime = (timestamp) => {
const dt = DateTime.fromSeconds(timestamp);
return dt.toFormat('y-MM-dd HH:mm')
}
}

2
routes/admin.vue
View File

@ -6,6 +6,8 @@
<T>admin.header</T>
</h2>
<p>Stats counted: {{$datetime(stats.calculatedAt)}}</p>
<section v-if="$isGranted('users')">
<details class="border mb-3" @click="loadUsers">
<summary class="bg-light p-3">

82
server/routes/admin.js
View File

@ -2,9 +2,10 @@ import { Router } from 'express';
import SQL from 'sql-template-strings';
import avatar from '../avatar';
import {config as socialLoginConfig} from "../social";
import {buildDict, now, shuffle, sortByValue, handleErrorAsync} from "../../src/helpers";
import {buildDict, now, shuffle, handleErrorAsync} from "../../src/helpers";
import locales from '../../src/locales';
import {decodeTime} from "ulid";
import {calculateStats, statsFile} from '../../src/stats';
import fs from 'fs';
const router = Router();
@ -98,85 +99,22 @@ router.get('/admin/users', handleErrorAsync(async (req, res) => {
return res.json(groupedUsers);
}));
const formatMonth = d => `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, '0')}-${d.getDate().toString().padStart(2, '0')}`;
const buildChart = (rows) => {
const dates = rows.map(row => new Date(decodeTime(row.id)));
const chart = {};
let loop = dates[0];
const end = dates[dates.length - 1];
while(loop <= end){
chart[formatMonth(loop)] = 0;
loop = new Date(loop.setDate(loop.getDate() + 1));
}
chart[formatMonth(loop)] = 0;
for (let date of dates) {
chart[formatMonth(date)]++;
}
return chart;
}
router.get('/admin/stats', handleErrorAsync(async (req, res) => {
if (!req.isGranted('panel')) {
return res.status(401).json({error: 'Unauthorised'});
}
const users = {
overall: (await req.db.get(SQL`SELECT count(*) AS c FROM users`)).c,
admins: (await req.db.get(SQL`SELECT count(*) AS c FROM users WHERE roles!=''`)).c,
chart: buildChart(await req.db.all(SQL`SELECT id FROM users ORDER BY id`)),
};
const stats = fs.existsSync(statsFile)
? JSON.parse(fs.readFileSync(statsFile))
: await calculateStats(req.db, req.locales);
const locales = {};
for (let locale in req.locales) {
if (!req.locales.hasOwnProperty(locale)) { continue; }
if (!req.isGranted('panel', locale)) { continue; }
const profiles = await req.db.all(SQL`SELECT pronouns, flags FROM profiles WHERE locale=${locale}`);
const pronouns = {}
const flags = {}
for (let profile of profiles) {
const pr = JSON.parse(profile.pronouns);
for (let pronoun in pr) {
if (!pr.hasOwnProperty(pronoun)) { continue; }
if (pronoun.includes(',') || pr[pronoun] < 0) {
continue;
}
const p = pronoun.replace(/^.*:\/\//, '').replace(/^\//, '').toLowerCase().replace(/^[a-z]+\.[^/]+\//, '');
if (pronouns[p] === undefined) {
pronouns[p] = 0;
}
pronouns[p] += pr[pronoun] === 1 ? 1 : 0.5;
}
const fl = JSON.parse(profile.flags);
for (let flag of fl) {
if (flags[flag] === undefined) {
flags[flag] = 0;
}
flags[flag] += 1;
}
for (let locale in stats.locales) {
if (stats.locales.hasOwnProperty(locale) && !req.isGranted('panel', locale)) {
delete stats.locales[locale];
}
locales[locale] = {
name: req.locales[locale].name,
url: req.locales[locale].url,
profiles: profiles.length,
pronouns: sortByValue(pronouns, true),
flags: sortByValue(flags, true),
nouns: {
approved: (await req.db.get(SQL`SELECT count(*) AS c FROM nouns WHERE locale=${locale} AND approved=1 AND deleted=0`)).c,
awaiting: (await req.db.get(SQL`SELECT count(*) AS c FROM nouns WHERE locale=${locale} AND approved=0 AND deleted=0`)).c,
},
chart: buildChart(await req.db.all(SQL`SELECT id FROM profiles WHERE locale=${locale} ORDER BY id`)),
};
}
return res.json({ users, locales });
return res.json(stats);
}));
export default router;

34
server/stats.js Normal file
View File

@ -0,0 +1,34 @@
const dbConnection = require('./db');
require('dotenv').config({ path:__dirname + '/../.env' });
const {calculateStats, statsFile} = require('../src/stats');
const locales = require('../src/locales');
const fs = require('fs');
// TODO duplication
const buildDict = (fn, ...args) => {
const dict = {};
for (let [key, value] of fn(...args)) {
dict[key] = value;
}
return dict;
}
const buildLocaleList = () => {
return buildDict(function* () {
for (let [code, name, url, published] of locales) {
if (published) {
yield [code, {name, url, published}];
}
}
})
}
async function calculate() {
const db = await dbConnection();
const stats = await calculateStats(db, buildLocaleList());
await db.close();
console.log(stats);
fs.writeFileSync(statsFile, JSON.stringify(stats));
}
calculate();

10
src/helpers.js
View File

@ -164,16 +164,6 @@ export const zip = (list, reverse) => {
});
}
export const sortByValue = (obj, reverse = false) => {
const sortedArray = [];
for (let i in obj) {
if (obj.hasOwnProperty(i)) {
sortedArray.push([parseInt(obj[i]), i]);
}
}
return zip(sortedArray.sort((a, b) => reverse ? b[0] - a[0] : a[0] - b[0]), true);
}
// https://stackoverflow.com/a/6274381/3297012
export const shuffle = a => {
for (let i = a.length - 1; i > 0; i--) {

2
src/locales.js
View File

@ -1,4 +1,4 @@
export default [
module.exports = [
['de', 'Deutsch', 'https://de.pronouns.page', false],
['es', 'Español', 'https://es.pronouns.page', true],
['en', 'English', 'https://en.pronouns.page', true],

112
src/stats.js Normal file
View File

@ -0,0 +1,112 @@
const {decodeTime} = require('ulid');
// TODO all the duplication...
const buildDict = (fn, ...args) => {
const dict = {};
for (let [key, value] of fn(...args)) {
dict[key] = value;
}
return dict;
}
const zip = (list, reverse) => {
return buildDict(function* () {
for (let [k, v] of list) {
yield reverse ? [v, k] : [k, v];
}
});
}
const sortByValue = (obj, reverse = false, firstN = -1) => {
let list = [];
for (let i in obj) {
if (obj.hasOwnProperty(i)) {
list.push([parseInt(obj[i]), i]);
}
}
list = list.sort((a, b) => reverse ? b[0] - a[0] : a[0] - b[0]);
if (firstN >= 0) {
list = list.slice(0, firstN);
}
return zip(list, true);
}
const formatMonth = d => `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, '0')}-${d.getDate().toString().padStart(2, '0')}`;
const buildChart = (rows) => {
const dates = rows.map(row => new Date(decodeTime(row.id)));
const chart = {};
let loop = dates[0];
const end = dates[dates.length - 1];
while(loop <= end){
chart[formatMonth(loop)] = 0;
loop = new Date(loop.setDate(loop.getDate() + 1));
}
chart[formatMonth(loop)] = 0;
for (let date of dates) {
chart[formatMonth(date)]++;
}
return chart;
}
module.exports.statsFile = process.env.STATS_FILE.replace('%projectdir%', __dirname + '/..')
module.exports.calculateStats = async (db, allLocales) => {
const users = {
overall: (await db.get(`SELECT count(*) AS c FROM users`)).c,
admins: (await db.get(`SELECT count(*) AS c FROM users WHERE roles!=''`)).c,
chart: buildChart(await db.all(`SELECT id FROM users ORDER BY id`)),
};
const locales = {};
for (let locale in allLocales) {
if (!allLocales.hasOwnProperty(locale)) { continue; }
const profiles = await db.all(`SELECT pronouns, flags FROM profiles WHERE locale='${locale}'`);
const pronouns = {}
const flags = {}
for (let profile of profiles) {
const pr = JSON.parse(profile.pronouns);
for (let pronoun in pr) {
if (!pr.hasOwnProperty(pronoun)) { continue; }
if (pronoun.includes(',') || pr[pronoun] < 0) {
continue;
}
const p = pronoun.replace(/^.*:\/\//, '').replace(/^\//, '').toLowerCase().replace(/^[a-z]+\.[^/]+\//, '');
if (pronouns[p] === undefined) {
pronouns[p] = 0;
}
pronouns[p] += pr[pronoun] === 1 ? 1 : 0.5;
}
const fl = JSON.parse(profile.flags);
for (let flag of fl) {
if (flags[flag] === undefined) {
flags[flag] = 0;
}
flags[flag] += 1;
}
}
locales[locale] = {
name: allLocales[locale].name,
url: allLocales[locale].url,
profiles: profiles.length,
pronouns: sortByValue(pronouns, true, 36),
flags: sortByValue(flags, true, 36),
nouns: {
approved: (await db.get(`SELECT count(*) AS c FROM nouns WHERE locale='${locale}' AND approved=1 AND deleted=0`)).c,
awaiting: (await db.get(`SELECT count(*) AS c FROM nouns WHERE locale='${locale}' AND approved=0 AND deleted=0`)).c,
},
chart: buildChart(await db.all(`SELECT id FROM profiles WHERE locale='${locale}' ORDER BY id`)),
};
}
return { calculatedAt: parseInt(new Date() / 1000), users, locales };
}