From 5c8c6eed633fe121ce686326935fb7f93c95e64d Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 17 Apr 2023 23:44:21 +0200 Subject: [PATCH] feat: add prometheus metrics --- backend/db/db.go | 13 ++++++++ backend/db/metrics.go | 63 ++++++++++++++++++++++++++++++++++++ backend/routes/meta/meta.go | 5 ++- backend/routes/mod/routes.go | 3 ++ backend/server/server.go | 13 +++++--- go.mod | 8 ++++- go.sum | 14 +++++++- 7 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 backend/db/metrics.go diff --git a/backend/db/db.go b/backend/db/db.go index 03caf25..25a214c 100644 --- a/backend/db/db.go +++ b/backend/db/db.go @@ -7,6 +7,7 @@ import ( "net/url" "os" + "codeberg.org/u1f320/pronouns.cc/backend/log" "emperror.dev/errors" "github.com/Masterminds/squirrel" "github.com/jackc/pgx/v5/pgconn" @@ -14,6 +15,7 @@ import ( "github.com/mediocregopher/radix/v4" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/prometheus/client_golang/prometheus" ) var sq = squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar) @@ -32,19 +34,24 @@ type DB struct { minio *minio.Client minioBucket string baseURL *url.URL + + TotalRequests prometheus.Counter } func New() (*DB, error) { + log.Debug("creating postgres client") pool, err := pgxpool.New(context.Background(), os.Getenv("DATABASE_URL")) if err != nil { return nil, errors.Wrap(err, "creating postgres client") } + log.Debug("creating redis client") redis, err := (&radix.PoolConfig{}).New(context.Background(), "tcp", os.Getenv("REDIS")) if err != nil { return nil, errors.Wrap(err, "creating redis client") } + log.Debug("creating minio client") minioClient, err := minio.New(os.Getenv("MINIO_ENDPOINT"), &minio.Options{ Creds: credentials.NewStaticV4(os.Getenv("MINIO_ACCESS_KEY_ID"), os.Getenv("MINIO_ACCESS_KEY_SECRET"), ""), Secure: os.Getenv("MINIO_SSL") == "true", @@ -67,6 +74,12 @@ func New() (*DB, error) { baseURL: baseURL, } + log.Debug("initializing metrics") + err = db.initMetrics() + if err != nil { + return nil, errors.Wrap(err, "initializing metrics") + } + return db, nil } diff --git a/backend/db/metrics.go b/backend/db/metrics.go new file mode 100644 index 0000000..144c3bc --- /dev/null +++ b/backend/db/metrics.go @@ -0,0 +1,63 @@ +package db + +import ( + "context" + + "codeberg.org/u1f320/pronouns.cc/backend/log" + "emperror.dev/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +func (db *DB) initMetrics() (err error) { + err = prometheus.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "pronouns_users_total", + Help: "The total number of registered users", + }, func() float64 { + count, err := db.TotalUserCount(context.Background()) + if err != nil { + log.Errorf("getting user count for metrics: %v", err) + } + return float64(count) + })) + if err != nil { + return errors.Wrap(err, "registering user count gauge") + } + + err = prometheus.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "pronouns_members_total", + Help: "The total number of registered members", + }, func() float64 { + count, err := db.TotalMemberCount(context.Background()) + if err != nil { + log.Errorf("getting member count for metrics: %v", err) + } + return float64(count) + })) + if err != nil { + return errors.Wrap(err, "registering member count gauge") + } + + db.TotalRequests = promauto.NewCounter(prometheus.CounterOpts{ + Name: "pronouns_api_requests_total", + Help: "The total number of API requests since the last restart", + }) + + return nil +} + +func (db *DB) TotalUserCount(ctx context.Context) (numUsers int64, err error) { + err = db.QueryRow(ctx, "SELECT COUNT(*) FROM users WHERE deleted_at IS NULL").Scan(&numUsers) + if err != nil { + return 0, errors.Wrap(err, "querying user count") + } + return numUsers, nil +} + +func (db *DB) TotalMemberCount(ctx context.Context) (numMembers int64, err error) { + err = db.QueryRow(ctx, "SELECT COUNT(*) FROM members WHERE unlisted = false AND user_id = ANY(SELECT id FROM users WHERE deleted_at IS NULL)").Scan(&numMembers) + if err != nil { + return 0, errors.Wrap(err, "querying member count") + } + return numMembers, nil +} diff --git a/backend/routes/meta/meta.go b/backend/routes/meta/meta.go index 40d27da..7b6c26e 100644 --- a/backend/routes/meta/meta.go +++ b/backend/routes/meta/meta.go @@ -31,13 +31,12 @@ type MetaResponse struct { func (s *Server) meta(w http.ResponseWriter, r *http.Request) error { ctx := r.Context() - var numUsers, numMembers int64 - err := s.DB.QueryRow(ctx, "SELECT COUNT(*) FROM users WHERE deleted_at IS NULL").Scan(&numUsers) + numUsers, err := s.DB.TotalUserCount(ctx) if err != nil { return errors.Wrap(err, "querying user count") } - err = s.DB.QueryRow(ctx, "SELECT COUNT(*) FROM members WHERE unlisted = false AND user_id = ANY(SELECT id FROM users WHERE deleted_at IS NULL)").Scan(&numMembers) + numMembers, err := s.DB.TotalMemberCount(ctx) if err != nil { return errors.Wrap(err, "querying user count") } diff --git a/backend/routes/mod/routes.go b/backend/routes/mod/routes.go index 52ff0aa..6aeef87 100644 --- a/backend/routes/mod/routes.go +++ b/backend/routes/mod/routes.go @@ -6,6 +6,7 @@ import ( "codeberg.org/u1f320/pronouns.cc/backend/server" "github.com/go-chi/chi/v5" "github.com/go-chi/render" + "github.com/prometheus/client_golang/prometheus/promhttp" ) type Server struct { @@ -23,6 +24,8 @@ func Mount(srv *server.Server, r chi.Router) { r.Patch("/reports/{id}", server.WrapHandler(s.resolveReport)) }) + r.With(MustAdmin).Handle("/metrics", promhttp.Handler()) + r.With(server.MustAuth).Post("/users/{id}/reports", server.WrapHandler(s.createUserReport)) r.With(server.MustAuth).Post("/members/{id}/reports", server.WrapHandler(s.createMemberReport)) diff --git a/backend/server/server.go b/backend/server/server.go index 459832d..3ed711c 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -107,12 +107,17 @@ func New() (*Server, error) { rateLimiter.Scope("*", "/auth/invites", 10) rateLimiter.Scope("POST", "/auth/discord/*", 10) - // rate limit handling - // - 120 req/minute (2/s) - // - keyed by Authorization header if valid token is provided, otherwise by IP - // - returns rate limit reset info in error s.Router.Use(rateLimiter.Handler()) + // increment the total requests counter whenever a request is made + s.Router.Use(func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + s.DB.TotalRequests.Inc() + next.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + }) + // return an API error for not found + method not allowed s.Router.NotFound(func(w http.ResponseWriter, r *http.Request) { render.Status(r, errCodeStatuses[ErrNotFound]) diff --git a/go.mod b/go.mod index 994a3a2..b335781 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/georgysavva/scany/v2 v2.0.0 github.com/go-chi/chi/v5 v5.0.8 + github.com/go-chi/cors v1.2.1 github.com/go-chi/httprate v0.7.1 github.com/go-chi/render v1.0.2 github.com/gobwas/glob v0.2.3 @@ -18,6 +19,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/mediocregopher/radix/v4 v4.1.2 github.com/minio/minio-go/v7 v7.0.50 + github.com/prometheus/client_golang v1.15.0 github.com/rs/xid v1.4.0 github.com/rubenv/sql-migrate v1.4.0 github.com/urfave/cli/v2 v2.25.1 @@ -27,10 +29,10 @@ require ( require ( github.com/ajg/form v1.5.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-chi/cors v1.2.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect @@ -43,11 +45,15 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/stretchr/objx v0.5.0 // indirect diff --git a/go.sum b/go.sum index 2f62af8..41cb19f 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bwmarrin/discordgo v0.27.1 h1:ib9AIc/dom1E/fSIulrBwnez0CToJE113ZGt4HoliGY= @@ -201,7 +203,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -336,6 +338,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/radix/v4 v4.1.2 h1:Pj7XnNK5WuzzFy63g98pnccainAePK+aZNQRvxSvj2I= github.com/mediocregopher/radix/v4 v4.1.2/go.mod h1:ajchozX/6ELmydxWeWM6xCFHVpZ4+67LXHOTOVR0nCE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -389,13 +393,21 @@ github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= +github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=