From bfe81deacce0c5ca62ff2b3213b9f1d501f45a45 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Mon, 18 Nov 2019 13:08:12 +0100 Subject: [PATCH] Add system metrics & stats Add repo, svc & rest endpoint for user, role & app metrics: total, deleted, suspended, archived, ... counters --- api/system/spec.json | 18 +++++++ api/system/spec/stats.json | 30 +++++++++++ docs/system/README.md | 25 ++++++++++ pkg/rh/metrics.go | 49 ++++++++++++++++++ system/repository/application.go | 23 +++++++++ system/repository/role.go | 44 ++++++++++++++++ system/repository/user.go | 44 ++++++++++++++++ system/rest/handlers/stats.go | 70 ++++++++++++++++++++++++++ system/rest/request/stats.go | 86 ++++++++++++++++++++++++++++++++ system/rest/router.go | 1 + system/rest/stats.go | 32 ++++++++++++ system/service/statistics.go | 73 +++++++++++++++++++++++++++ system/types/applications.go | 6 +++ system/types/role.go | 11 ++++ system/types/user.go | 11 ++++ 15 files changed, 523 insertions(+) create mode 100644 api/system/spec/stats.json create mode 100644 pkg/rh/metrics.go create mode 100644 system/rest/handlers/stats.go create mode 100644 system/rest/request/stats.go create mode 100644 system/rest/stats.go create mode 100644 system/service/statistics.go diff --git a/api/system/spec.json b/api/system/spec.json index ad19ab0b3..d5c624d04 100644 --- a/api/system/spec.json +++ b/api/system/spec.json @@ -1971,5 +1971,23 @@ } } ] + }, + { + "title": "Statistics", + "entrypoint": "stats", + "path": "/stats", + "authentication": [ + "Client ID", + "Session ID" + ], + "apis": [ + { + "name": "list", + "method": "GET", + "title": "List system statistics", + "path": "/", + "parameters": {} + } + ] } ] diff --git a/api/system/spec/stats.json b/api/system/spec/stats.json new file mode 100644 index 000000000..58a05f05b --- /dev/null +++ b/api/system/spec/stats.json @@ -0,0 +1,30 @@ +{ + "Title": "Statistics", + "Interface": "Stats", + "Struct": null, + "Parameters": null, + "Protocol": "", + "Authentication": [ + "Client ID", + "Session ID" + ], + "Path": "/stats", + "APIs": [ + { + "Name": "list", + "Method": "GET", + "Title": "List system statistics", + "Path": "/", + "Parameters": { + "get": [ + { + "name": "metrics", + "required": false, + "title": "Get only specified metrics", + "type": "[]string" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/docs/system/README.md b/docs/system/README.md index 6b97f9b97..ecf54c1d3 100644 --- a/docs/system/README.md +++ b/docs/system/README.md @@ -1133,6 +1133,31 @@ An organisation may have many roles. Roles may have many channels available. Acc +# Statistics + +| Method | Endpoint | Purpose | +| ------ | -------- | ------- | +| `GET` | `/stats/` | List system statistics | + +## List system statistics + +#### Method + +| URI | Protocol | Method | Authentication | +| --- | -------- | ------ | -------------- | +| `/stats/` | HTTP/S | GET | Client ID, Session ID | + +#### Request parameters + +| Parameter | Type | Method | Description | Default | Required? | +| --------- | ---- | ------ | ----------- | ------- | --------- | +| metrics | []string | GET | Get only specified metrics | N/A | NO | + +--- + + + + # Subscription | Method | Endpoint | Purpose | diff --git a/pkg/rh/metrics.go b/pkg/rh/metrics.go new file mode 100644 index 000000000..fd51a73fa --- /dev/null +++ b/pkg/rh/metrics.go @@ -0,0 +1,49 @@ +package rh + +import ( + "fmt" + + "github.com/Masterminds/squirrel" + "github.com/titpetric/factory" +) + +// DailyMetrics aids repositories on simple stat building queries +// +// Returns a slice of numbers (timestamp + value pairs +func DailyMetrics(db *factory.DB, q squirrel.SelectBuilder, field string) (rval []uint, err error) { + var ( + aux = make([]struct { + Timestamp uint + Value uint + }, 0) + ) + + q = q. + Column(fmt.Sprintf("UNIX_TIMESTAMP(DATE(%s)) timestamp", field)). + Column("COUNT(*) AS value"). + Where(fmt.Sprintf("%s IS NOT NULL", field)). + GroupBy("timestamp") + + if err = FetchAll(db, q, &aux); err != nil { + return + } + + rval = make([]uint, 2*len(aux)) + for i := 0; i < len(aux); i++ { + rval[2*i], rval[2*i+1] = aux[i].Timestamp, aux[i].Value + } + + return +} + +// MultiDailyMetrics simplifies fetching of multiple daily metrics +func MultiDailyMetrics(db *factory.DB, q squirrel.SelectBuilder, fields []string, mm ...*[]uint) (err error) { + for m := 0; m < len(mm); m++ { + *mm[m], err = DailyMetrics(db, q, fields[m]) + if err != nil { + return + } + } + + return +} diff --git a/system/repository/application.go b/system/repository/application.go index aea33c1ae..303e9a1ef 100644 --- a/system/repository/application.go +++ b/system/repository/application.go @@ -23,6 +23,8 @@ type ( DeleteByID(id uint64) error UndeleteByID(id uint64) error + + Metrics() (*types.ApplicationMetrics, error) } application struct { @@ -147,3 +149,24 @@ func (r *application) DeleteByID(id uint64) error { func (r *application) UndeleteByID(id uint64) error { return rh.UpdateColumns(r.db(), r.table(), rh.Set{"deleted_at": nil}, squirrel.Eq{"id": id}) } + +// Metrics collects and returns user metrics +func (r application) Metrics() (rval *types.ApplicationMetrics, err error) { + var ( + counters = squirrel. + Select( + "COUNT(*) as total", + "SUM(IF(deleted_at IS NULL, 0, 1)) as deleted", + "SUM(IF(deleted_at IS NULL, 1, 0)) as valid", + ). + From(r.table() + " AS u") + ) + + rval = &types.ApplicationMetrics{} + + if err = rh.FetchOne(r.db(), counters, rval); err != nil { + return + } + + return +} diff --git a/system/repository/role.go b/system/repository/role.go index cb9a8f73a..19435e78c 100644 --- a/system/repository/role.go +++ b/system/repository/role.go @@ -35,6 +35,8 @@ type ( MemberFindByRoleID(roleID uint64) ([]*types.RoleMember, error) MemberAddByID(roleID, userID uint64) error MemberRemoveByID(roleID, userID uint64) error + + Metrics() (*types.RoleMetrics, error) } role struct { @@ -235,3 +237,45 @@ func (r *role) MemberRemoveByID(roleID, userID uint64) error { } return r.db().Delete(r.tableMember(), mod, "rel_role", "rel_user") } + +// Metrics collects and returns user metrics +func (r role) Metrics() (rval *types.RoleMetrics, err error) { + var ( + counters = squirrel. + Select( + "COUNT(*) as total", + "SUM(IF(deleted_at IS NULL, 0, 1)) as deleted", + "SUM(IF(archived_at IS NULL, 0, 1)) as archived", + "SUM(IF(deleted_at IS NULL AND archived_at IS NULL, 1, 0)) as valid", + ). + From(r.table() + " AS u") + ) + + rval = &types.RoleMetrics{} + + if err = rh.FetchOne(r.db(), counters, rval); err != nil { + return + } + + // Fetch daily metrics for created, updated, deleted and archived roles + err = rh.MultiDailyMetrics( + r.db(), + squirrel.Select().From(r.table()+" AS u"), + []string{ + "created_at", + "updated_at", + "deleted_at", + "archived_at", + }, + &rval.DailyCreated, + &rval.DailyUpdated, + &rval.DailyDeleted, + &rval.DailyArchived, + ) + + if err != nil { + return + } + + return +} diff --git a/system/repository/user.go b/system/repository/user.go index 7fe8a7770..54d609b7f 100644 --- a/system/repository/user.go +++ b/system/repository/user.go @@ -33,6 +33,8 @@ type ( UnsuspendByID(id uint64) error DeleteByID(id uint64) error UndeleteByID(id uint64) error + + Metrics() (*types.UserMetrics, error) } user struct { @@ -237,3 +239,45 @@ func (r *user) DeleteByID(id uint64) error { func (r *user) UndeleteByID(id uint64) error { return rh.UpdateColumns(r.db(), r.table(), rh.Set{"deleted_at": nil}, squirrel.Eq{"id": id}) } + +// Metrics collects and returns user metrics +func (r user) Metrics() (rval *types.UserMetrics, err error) { + var ( + counters = squirrel. + Select( + "COUNT(*) as total", + "SUM(IF(deleted_at IS NULL, 0, 1)) as deleted", + "SUM(IF(suspended_at IS NULL, 0, 1)) as suspended", + "SUM(IF(deleted_at IS NULL AND suspended_at IS NULL, 1, 0)) as valid", + ). + From(r.table() + " AS u") + ) + + rval = &types.UserMetrics{} + + if err = rh.FetchOne(r.db(), counters, rval); err != nil { + return + } + + // Fetch daily metrics for created, updated, deleted and suspended users + err = rh.MultiDailyMetrics( + r.db(), + squirrel.Select().From(r.table()+" AS u"), + []string{ + "created_at", + "updated_at", + "deleted_at", + "suspended_at", + }, + &rval.DailyCreated, + &rval.DailyUpdated, + &rval.DailyDeleted, + &rval.DailySuspended, + ) + + if err != nil { + return + } + + return +} diff --git a/system/rest/handlers/stats.go b/system/rest/handlers/stats.go new file mode 100644 index 000000000..5b221a957 --- /dev/null +++ b/system/rest/handlers/stats.go @@ -0,0 +1,70 @@ +package handlers + +/* + Hello! This file is auto-generated from `docs/src/spec.json`. + + For development: + In order to update the generated files, edit this file under the location, + add your struct fields, imports, API definitions and whatever you want, and: + + 1. run [spec](https://github.com/titpetric/spec) in the same folder, + 2. run `./_gen.php` in this folder. + + You may edit `stats.go`, `stats.util.go` or `stats_test.go` to + implement your API calls, helper functions and tests. The file `stats.go` + is only generated the first time, and will not be overwritten if it exists. +*/ + +import ( + "context" + + "net/http" + + "github.com/go-chi/chi" + "github.com/titpetric/factory/resputil" + + "github.com/cortezaproject/corteza-server/pkg/logger" + "github.com/cortezaproject/corteza-server/system/rest/request" +) + +// Internal API interface +type StatsAPI interface { + List(context.Context, *request.StatsList) (interface{}, error) +} + +// HTTP API interface +type Stats struct { + List func(http.ResponseWriter, *http.Request) +} + +func NewStats(h StatsAPI) *Stats { + return &Stats{ + List: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewStatsList() + if err := params.Fill(r); err != nil { + logger.LogParamError("Stats.List", r, err) + resputil.JSON(w, err) + return + } + + value, err := h.List(r.Context(), params) + if err != nil { + logger.LogControllerError("Stats.List", r, err, params.Auditable()) + resputil.JSON(w, err) + return + } + logger.LogControllerCall("Stats.List", r, params.Auditable()) + if !serveHTTP(value, w, r) { + resputil.JSON(w, value) + } + }, + } +} + +func (h Stats) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) { + r.Group(func(r chi.Router) { + r.Use(middlewares...) + r.Get("/stats/", h.List) + }) +} diff --git a/system/rest/request/stats.go b/system/rest/request/stats.go new file mode 100644 index 000000000..04189882c --- /dev/null +++ b/system/rest/request/stats.go @@ -0,0 +1,86 @@ +package request + +/* + Hello! This file is auto-generated from `docs/src/spec.json`. + + For development: + In order to update the generated files, edit this file under the location, + add your struct fields, imports, API definitions and whatever you want, and: + + 1. run [spec](https://github.com/titpetric/spec) in the same folder, + 2. run `./_gen.php` in this folder. + + You may edit `stats.go`, `stats.util.go` or `stats_test.go` to + implement your API calls, helper functions and tests. The file `stats.go` + is only generated the first time, and will not be overwritten if it exists. +*/ + +import ( + "io" + "strings" + + "encoding/json" + "mime/multipart" + "net/http" + + "github.com/go-chi/chi" + "github.com/pkg/errors" +) + +var _ = chi.URLParam +var _ = multipart.FileHeader{} + +// Stats list request parameters +type StatsList struct { + Metrics []string +} + +func NewStatsList() *StatsList { + return &StatsList{} +} + +func (r StatsList) Auditable() map[string]interface{} { + var out = map[string]interface{}{} + + out["metrics"] = r.Metrics + + return out +} + +func (r *StatsList) Fill(req *http.Request) (err error) { + if strings.ToLower(req.Header.Get("content-type")) == "application/json" { + err = json.NewDecoder(req.Body).Decode(r) + + switch { + case err == io.EOF: + err = nil + case err != nil: + return errors.Wrap(err, "error parsing http request body") + } + } + + if err = req.ParseForm(); err != nil { + return err + } + + get := map[string]string{} + post := map[string]string{} + urlQuery := req.URL.Query() + for name, param := range urlQuery { + get[name] = string(param[0]) + } + postVars := req.Form + for name, param := range postVars { + post[name] = string(param[0]) + } + + if val, ok := urlQuery["metrics[]"]; ok { + r.Metrics = parseStrings(val) + } else if val, ok = urlQuery["metrics"]; ok { + r.Metrics = parseStrings(val) + } + + return err +} + +var _ RequestFiller = NewStatsList() diff --git a/system/rest/router.go b/system/rest/router.go index 4d6b0b17e..583f2a696 100644 --- a/system/rest/router.go +++ b/system/rest/router.go @@ -34,6 +34,7 @@ func MountRoutes(r chi.Router) { handlers.NewPermissions(Permissions{}.New()).MountRoutes(r) handlers.NewApplication(Application{}.New()).MountRoutes(r) handlers.NewSettings(Settings{}.New()).MountRoutes(r) + handlers.NewStats(Stats{}.New()).MountRoutes(r) handlers.NewAutomationScript(AutomationScript{}.New()).MountRoutes(r) handlers.NewAutomationTrigger(AutomationTrigger{}.New()).MountRoutes(r) diff --git a/system/rest/stats.go b/system/rest/stats.go new file mode 100644 index 000000000..04bcdd913 --- /dev/null +++ b/system/rest/stats.go @@ -0,0 +1,32 @@ +package rest + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/cortezaproject/corteza-server/system/rest/request" + "github.com/cortezaproject/corteza-server/system/service" +) + +var _ = errors.Wrap + +type ( + Stats struct { + svc statsService + } + + statsService interface { + Metrics(context.Context) (interface{}, error) + } +) + +func (Stats) New() *Stats { + return &Stats{ + svc: service.Statistics(context.Background()), + } +} + +func (ctrl *Stats) List(ctx context.Context, r *request.StatsList) (interface{}, error) { + return ctrl.svc.Metrics(ctx) +} diff --git a/system/service/statistics.go b/system/service/statistics.go new file mode 100644 index 000000000..1def794e3 --- /dev/null +++ b/system/service/statistics.go @@ -0,0 +1,73 @@ +package service + +import ( + "context" + + "github.com/titpetric/factory" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/cortezaproject/corteza-server/pkg/logger" + "github.com/cortezaproject/corteza-server/system/repository" + "github.com/cortezaproject/corteza-server/system/types" +) + +type ( + statistics struct { + db *factory.DB + ctx context.Context + logger *zap.Logger + + ac statisticsAccessController + } + + statisticsAccessController interface { + CanAccess(context.Context) bool + } +) + +func Statistics(ctx context.Context) *statistics { + return &statistics{ + db: repository.DB(ctx), + ac: DefaultAccessControl, + logger: DefaultLogger.Named("statistics"), + } +} + +// log() returns zap's logger with requestID from current context and fields. +func (svc statistics) log(ctx context.Context, fields ...zapcore.Field) *zap.Logger { + return logger.AddRequestID(ctx, svc.logger).With(fields...) +} + +func (svc statistics) Metrics(ctx context.Context) (interface{}, error) { + if !svc.ac.CanAccess(ctx) { + return nil, ErrNoPermissions + } + + type ( + metricsPayload struct { + Users *types.UserMetrics `json:"users"` + Roles *types.RoleMetrics `json:"roles"` + Applications *types.ApplicationMetrics `json:"applications"` + } + ) + + var ( + rval = &metricsPayload{} + err error + ) + + if rval.Users, err = repository.User(ctx, svc.db).Metrics(); err != nil { + return nil, err + } + + if rval.Roles, err = repository.Role(ctx, svc.db).Metrics(); err != nil { + return nil, err + } + + if rval.Applications, err = repository.Application(ctx, svc.db).Metrics(); err != nil { + return nil, err + } + + return rval, err +} diff --git a/system/types/applications.go b/system/types/applications.go index 08f030ca2..8113e32ed 100644 --- a/system/types/applications.go +++ b/system/types/applications.go @@ -49,6 +49,12 @@ type ( Deleted rh.FilterState `json:"deleted"` } + + ApplicationMetrics struct { + Total uint `json:"total"` + Deleted uint `json:"deleted"` + Valid uint `json:"valid"` + } ) func (a *Application) Valid() bool { diff --git a/system/types/role.go b/system/types/role.go index a58441731..b3c9eec6e 100644 --- a/system/types/role.go +++ b/system/types/role.go @@ -39,6 +39,17 @@ type ( // Resource permission check filter IsReadable *permissions.ResourceFilter `json:"-"` } + + RoleMetrics struct { + Total uint `json:"total"` + Valid uint `json:"valid"` + Deleted uint `json:"deleted"` + Archived uint `json:"archived"` + DailyCreated []uint `json:"dailyCreated"` + DailyDeleted []uint `json:"dailyDeleted"` + DailyUpdated []uint `json:"dailyUpdated"` + DailyArchived []uint `json:"dailyArchived"` + } ) // Resource returns a resource ID for this type diff --git a/system/types/user.go b/system/types/user.go index d4e4ff4b1..ed5e4ce7d 100644 --- a/system/types/user.go +++ b/system/types/user.go @@ -71,6 +71,17 @@ type ( } UserKind string + + UserMetrics struct { + Total uint `json:"total"` + Valid uint `json:"valid"` + Deleted uint `json:"deleted"` + Suspended uint `json:"suspended"` + DailyCreated []uint `json:"dailyCreated"` + DailyDeleted []uint `json:"dailyDeleted"` + DailyUpdated []uint `json:"dailyUpdated"` + DailySuspended []uint `json:"dailySuspended"` + } ) const (