Add system metrics & stats

Add repo, svc & rest endpoint for user, role & app metrics:
total, deleted, suspended, archived, ... counters

api/system/spec.json

@@ -1971,5 +1971,23 @@
         }
       }
     ]
+  },
+  {
+    "title": "Statistics",
+    "entrypoint": "stats",
+    "path": "/stats",
+    "authentication": [
+      "Client ID",
+      "Session ID"
+    ],
+    "apis": [
+      {
+        "name": "list",
+        "method": "GET",
+        "title": "List system statistics",
+        "path": "/",
+        "parameters": {}
+      }
+    ]
   }
 ]

api/system/spec/stats.json

@@ -0,0 +1,30 @@
+{
+  "Title": "Statistics",
+  "Interface": "Stats",
+  "Struct": null,
+  "Parameters": null,
+  "Protocol": "",
+  "Authentication": [
+    "Client ID",
+    "Session ID"
+  ],
+  "Path": "/stats",
+  "APIs": [
+    {
+      "Name": "list",
+      "Method": "GET",
+      "Title": "List system statistics",
+      "Path": "/",
+      "Parameters": {
+        "get": [
+          {
+            "name": "metrics",
+            "required": false,
+            "title": "Get only specified metrics",
+            "type": "[]string"
+          }
+        ]
+      }
+    }
+  ]
+}

docs/system/README.md

@@ -1133,6 +1133,31 @@ An organisation may have many roles. Roles may have many channels available. Acc
 
 
 
+# Statistics
+
+| Method | Endpoint | Purpose |
+| ------ | -------- | ------- |
+| `GET` | `/stats/` | List system statistics |
+
+## List system statistics
+
+#### Method
+
+| URI | Protocol | Method | Authentication |
+| --- | -------- | ------ | -------------- |
+| `/stats/` | HTTP/S | GET | Client ID, Session ID |
+
+#### Request parameters
+
+| Parameter | Type | Method | Description | Default | Required? |
+| --------- | ---- | ------ | ----------- | ------- | --------- |
+| metrics | []string | GET | Get only specified metrics | N/A | NO |
+
+---
+
+
+
+
 # Subscription
 
 | Method | Endpoint | Purpose |

pkg/rh/metrics.go

@@ -0,0 +1,49 @@
+package rh
+
+import (
+	"fmt"
+
+	"github.com/Masterminds/squirrel"
+	"github.com/titpetric/factory"
+)
+
+// DailyMetrics aids repositories on simple stat building queries
+//
+// Returns a slice of numbers (timestamp + value pairs
+func DailyMetrics(db *factory.DB, q squirrel.SelectBuilder, field string) (rval []uint, err error) {
+	var (
+		aux = make([]struct {
+			Timestamp uint
+			Value     uint
+		}, 0)
+	)
+
+	q = q.
+		Column(fmt.Sprintf("UNIX_TIMESTAMP(DATE(%s)) timestamp", field)).
+		Column("COUNT(*) AS value").
+		Where(fmt.Sprintf("%s IS NOT NULL", field)).
+		GroupBy("timestamp")
+
+	if err = FetchAll(db, q, &aux); err != nil {
+		return
+	}
+
+	rval = make([]uint, 2*len(aux))
+	for i := 0; i < len(aux); i++ {
+		rval[2*i], rval[2*i+1] = aux[i].Timestamp, aux[i].Value
+	}
+
+	return
+}
+
+// MultiDailyMetrics simplifies fetching of multiple daily metrics
+func MultiDailyMetrics(db *factory.DB, q squirrel.SelectBuilder, fields []string, mm ...*[]uint) (err error) {
+	for m := 0; m < len(mm); m++ {
+		*mm[m], err = DailyMetrics(db, q, fields[m])
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

system/repository/application.go

@@ -23,6 +23,8 @@ type (
 
 		DeleteByID(id uint64) error
 		UndeleteByID(id uint64) error
+
+		Metrics() (*types.ApplicationMetrics, error)
 	}
 
 	application struct {
@@ -147,3 +149,24 @@ func (r *application) DeleteByID(id uint64) error {
 func (r *application) UndeleteByID(id uint64) error {
 	return rh.UpdateColumns(r.db(), r.table(), rh.Set{"deleted_at": nil}, squirrel.Eq{"id": id})
 }
+
+// Metrics collects and returns user metrics
+func (r application) Metrics() (rval *types.ApplicationMetrics, err error) {
+	var (
+		counters = squirrel.
+			Select(
+				"COUNT(*) as total",
+				"SUM(IF(deleted_at IS NULL, 0, 1)) as deleted",
+				"SUM(IF(deleted_at IS NULL, 1, 0)) as valid",
+			).
+			From(r.table() + " AS u")
+	)
+
+	rval = &types.ApplicationMetrics{}
+
+	if err = rh.FetchOne(r.db(), counters, rval); err != nil {
+		return
+	}
+
+	return
+}

system/repository/role.go

@@ -35,6 +35,8 @@ type (
 		MemberFindByRoleID(roleID uint64) ([]*types.RoleMember, error)
 		MemberAddByID(roleID, userID uint64) error
 		MemberRemoveByID(roleID, userID uint64) error
+
+		Metrics() (*types.RoleMetrics, error)
 	}
 
 	role struct {
@@ -235,3 +237,45 @@ func (r *role) MemberRemoveByID(roleID, userID uint64) error {
 	}
 	return r.db().Delete(r.tableMember(), mod, "rel_role", "rel_user")
 }
+
+// Metrics collects and returns user metrics
+func (r role) Metrics() (rval *types.RoleMetrics, err error) {
+	var (
+		counters = squirrel.
+			Select(
+				"COUNT(*) as total",
+				"SUM(IF(deleted_at IS NULL, 0, 1)) as deleted",
+				"SUM(IF(archived_at IS NULL, 0, 1)) as archived",
+				"SUM(IF(deleted_at IS NULL AND archived_at IS NULL, 1, 0)) as valid",
+			).
+			From(r.table() + " AS u")
+	)
+
+	rval = &types.RoleMetrics{}
+
+	if err = rh.FetchOne(r.db(), counters, rval); err != nil {
+		return
+	}
+
+	// Fetch daily metrics for created, updated, deleted and archived roles
+	err = rh.MultiDailyMetrics(
+		r.db(),
+		squirrel.Select().From(r.table()+" AS u"),
+		[]string{
+			"created_at",
+			"updated_at",
+			"deleted_at",
+			"archived_at",
+		},
+		&rval.DailyCreated,
+		&rval.DailyUpdated,
+		&rval.DailyDeleted,
+		&rval.DailyArchived,
+	)
+
+	if err != nil {
+		return
+	}
+
+	return
+}

system/repository/user.go

@@ -33,6 +33,8 @@ type (
 		UnsuspendByID(id uint64) error
 		DeleteByID(id uint64) error
 		UndeleteByID(id uint64) error
+
+		Metrics() (*types.UserMetrics, error)
 	}
 
 	user struct {
@@ -237,3 +239,45 @@ func (r *user) DeleteByID(id uint64) error {
 func (r *user) UndeleteByID(id uint64) error {
 	return rh.UpdateColumns(r.db(), r.table(), rh.Set{"deleted_at": nil}, squirrel.Eq{"id": id})
 }
+
+// Metrics collects and returns user metrics
+func (r user) Metrics() (rval *types.UserMetrics, err error) {
+	var (
+		counters = squirrel.
+			Select(
+				"COUNT(*) as total",
+				"SUM(IF(deleted_at IS NULL, 0, 1)) as deleted",
+				"SUM(IF(suspended_at IS NULL, 0, 1)) as suspended",
+				"SUM(IF(deleted_at IS NULL AND suspended_at IS NULL, 1, 0)) as valid",
+			).
+			From(r.table() + " AS u")
+	)
+
+	rval = &types.UserMetrics{}
+
+	if err = rh.FetchOne(r.db(), counters, rval); err != nil {
+		return
+	}
+
+	// Fetch daily metrics for created, updated, deleted and suspended users
+	err = rh.MultiDailyMetrics(
+		r.db(),
+		squirrel.Select().From(r.table()+" AS u"),
+		[]string{
+			"created_at",
+			"updated_at",
+			"deleted_at",
+			"suspended_at",
+		},
+		&rval.DailyCreated,
+		&rval.DailyUpdated,
+		&rval.DailyDeleted,
+		&rval.DailySuspended,
+	)
+
+	if err != nil {
+		return
+	}
+
+	return
+}

system/rest/handlers/stats.go

@@ -0,0 +1,70 @@
+package handlers
+
+/*
+	Hello! This file is auto-generated from `docs/src/spec.json`.
+
+	For development:
+	In order to update the generated files, edit this file under the location,
+	add your struct fields, imports, API definitions and whatever you want, and:
+
+	1. run [spec](https://github.com/titpetric/spec) in the same folder,
+	2. run `./_gen.php` in this folder.
+
+	You may edit `stats.go`, `stats.util.go` or `stats_test.go` to
+	implement your API calls, helper functions and tests. The file `stats.go`
+	is only generated the first time, and will not be overwritten if it exists.
+*/
+
+import (
+	"context"
+
+	"net/http"
+
+	"github.com/go-chi/chi"
+	"github.com/titpetric/factory/resputil"
+
+	"github.com/cortezaproject/corteza-server/pkg/logger"
+	"github.com/cortezaproject/corteza-server/system/rest/request"
+)
+
+// Internal API interface
+type StatsAPI interface {
+	List(context.Context, *request.StatsList) (interface{}, error)
+}
+
+// HTTP API interface
+type Stats struct {
+	List func(http.ResponseWriter, *http.Request)
+}
+
+func NewStats(h StatsAPI) *Stats {
+	return &Stats{
+		List: func(w http.ResponseWriter, r *http.Request) {
+			defer r.Body.Close()
+			params := request.NewStatsList()
+			if err := params.Fill(r); err != nil {
+				logger.LogParamError("Stats.List", r, err)
+				resputil.JSON(w, err)
+				return
+			}
+
+			value, err := h.List(r.Context(), params)
+			if err != nil {
+				logger.LogControllerError("Stats.List", r, err, params.Auditable())
+				resputil.JSON(w, err)
+				return
+			}
+			logger.LogControllerCall("Stats.List", r, params.Auditable())
+			if !serveHTTP(value, w, r) {
+				resputil.JSON(w, value)
+			}
+		},
+	}
+}
+
+func (h Stats) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) {
+	r.Group(func(r chi.Router) {
+		r.Use(middlewares...)
+		r.Get("/stats/", h.List)
+	})
+}

system/rest/request/stats.go

@@ -0,0 +1,86 @@
+package request
+
+/*
+	Hello! This file is auto-generated from `docs/src/spec.json`.
+
+	For development:
+	In order to update the generated files, edit this file under the location,
+	add your struct fields, imports, API definitions and whatever you want, and:
+
+	1. run [spec](https://github.com/titpetric/spec) in the same folder,
+	2. run `./_gen.php` in this folder.
+
+	You may edit `stats.go`, `stats.util.go` or `stats_test.go` to
+	implement your API calls, helper functions and tests. The file `stats.go`
+	is only generated the first time, and will not be overwritten if it exists.
+*/
+
+import (
+	"io"
+	"strings"
+
+	"encoding/json"
+	"mime/multipart"
+	"net/http"
+
+	"github.com/go-chi/chi"
+	"github.com/pkg/errors"
+)
+
+var _ = chi.URLParam
+var _ = multipart.FileHeader{}
+
+// Stats list request parameters
+type StatsList struct {
+	Metrics []string
+}
+
+func NewStatsList() *StatsList {
+	return &StatsList{}
+}
+
+func (r StatsList) Auditable() map[string]interface{} {
+	var out = map[string]interface{}{}
+
+	out["metrics"] = r.Metrics
+
+	return out
+}
+
+func (r *StatsList) Fill(req *http.Request) (err error) {
+	if strings.ToLower(req.Header.Get("content-type")) == "application/json" {
+		err = json.NewDecoder(req.Body).Decode(r)
+
+		switch {
+		case err == io.EOF:
+			err = nil
+		case err != nil:
+			return errors.Wrap(err, "error parsing http request body")
+		}
+	}
+
+	if err = req.ParseForm(); err != nil {
+		return err
+	}
+
+	get := map[string]string{}
+	post := map[string]string{}
+	urlQuery := req.URL.Query()
+	for name, param := range urlQuery {
+		get[name] = string(param[0])
+	}
+	postVars := req.Form
+	for name, param := range postVars {
+		post[name] = string(param[0])
+	}
+
+	if val, ok := urlQuery["metrics[]"]; ok {
+		r.Metrics = parseStrings(val)
+	} else if val, ok = urlQuery["metrics"]; ok {
+		r.Metrics = parseStrings(val)
+	}
+
+	return err
+}
+
+var _ RequestFiller = NewStatsList()

system/rest/router.go

@@ -34,6 +34,7 @@ func MountRoutes(r chi.Router) {
 		handlers.NewPermissions(Permissions{}.New()).MountRoutes(r)
 		handlers.NewApplication(Application{}.New()).MountRoutes(r)
 		handlers.NewSettings(Settings{}.New()).MountRoutes(r)
+		handlers.NewStats(Stats{}.New()).MountRoutes(r)
 
 		handlers.NewAutomationScript(AutomationScript{}.New()).MountRoutes(r)
 		handlers.NewAutomationTrigger(AutomationTrigger{}.New()).MountRoutes(r)

system/rest/stats.go

@@ -0,0 +1,32 @@
+package rest
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/cortezaproject/corteza-server/system/rest/request"
+	"github.com/cortezaproject/corteza-server/system/service"
+)
+
+var _ = errors.Wrap
+
+type (
+	Stats struct {
+		svc statsService
+	}
+
+	statsService interface {
+		Metrics(context.Context) (interface{}, error)
+	}
+)
+
+func (Stats) New() *Stats {
+	return &Stats{
+		svc: service.Statistics(context.Background()),
+	}
+}
+
+func (ctrl *Stats) List(ctx context.Context, r *request.StatsList) (interface{}, error) {
+	return ctrl.svc.Metrics(ctx)
+}

system/service/statistics.go

@@ -0,0 +1,73 @@
+package service
+
+import (
+	"context"
+
+	"github.com/titpetric/factory"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/cortezaproject/corteza-server/pkg/logger"
+	"github.com/cortezaproject/corteza-server/system/repository"
+	"github.com/cortezaproject/corteza-server/system/types"
+)
+
+type (
+	statistics struct {
+		db     *factory.DB
+		ctx    context.Context
+		logger *zap.Logger
+
+		ac statisticsAccessController
+	}
+
+	statisticsAccessController interface {
+		CanAccess(context.Context) bool
+	}
+)
+
+func Statistics(ctx context.Context) *statistics {
+	return &statistics{
+		db:     repository.DB(ctx),
+		ac:     DefaultAccessControl,
+		logger: DefaultLogger.Named("statistics"),
+	}
+}
+
+// log() returns zap's logger with requestID from current context and fields.
+func (svc statistics) log(ctx context.Context, fields ...zapcore.Field) *zap.Logger {
+	return logger.AddRequestID(ctx, svc.logger).With(fields...)
+}
+
+func (svc statistics) Metrics(ctx context.Context) (interface{}, error) {
+	if !svc.ac.CanAccess(ctx) {
+		return nil, ErrNoPermissions
+	}
+
+	type (
+		metricsPayload struct {
+			Users        *types.UserMetrics        `json:"users"`
+			Roles        *types.RoleMetrics        `json:"roles"`
+			Applications *types.ApplicationMetrics `json:"applications"`
+		}
+	)
+
+	var (
+		rval = &metricsPayload{}
+		err  error
+	)
+
+	if rval.Users, err = repository.User(ctx, svc.db).Metrics(); err != nil {
+		return nil, err
+	}
+
+	if rval.Roles, err = repository.Role(ctx, svc.db).Metrics(); err != nil {
+		return nil, err
+	}
+
+	if rval.Applications, err = repository.Application(ctx, svc.db).Metrics(); err != nil {
+		return nil, err
+	}
+
+	return rval, err
+}

system/types/applications.go

@@ -49,6 +49,12 @@ type (
 
 		Deleted rh.FilterState `json:"deleted"`
 	}
+
+	ApplicationMetrics struct {
+		Total   uint `json:"total"`
+		Deleted uint `json:"deleted"`
+		Valid   uint `json:"valid"`
+	}
 )
 
 func (a *Application) Valid() bool {

system/types/role.go

@@ -39,6 +39,17 @@ type (
 		// Resource permission check filter
 		IsReadable *permissions.ResourceFilter `json:"-"`
 	}
+
+	RoleMetrics struct {
+		Total         uint   `json:"total"`
+		Valid         uint   `json:"valid"`
+		Deleted       uint   `json:"deleted"`
+		Archived      uint   `json:"archived"`
+		DailyCreated  []uint `json:"dailyCreated"`
+		DailyDeleted  []uint `json:"dailyDeleted"`
+		DailyUpdated  []uint `json:"dailyUpdated"`
+		DailyArchived []uint `json:"dailyArchived"`
+	}
 )
 
 // Resource returns a resource ID for this type

system/types/user.go

@@ -71,6 +71,17 @@ type (
 	}
 
 	UserKind string
+
+	UserMetrics struct {
+		Total          uint   `json:"total"`
+		Valid          uint   `json:"valid"`
+		Deleted        uint   `json:"deleted"`
+		Suspended      uint   `json:"suspended"`
+		DailyCreated   []uint `json:"dailyCreated"`
+		DailyDeleted   []uint `json:"dailyDeleted"`
+		DailyUpdated   []uint `json:"dailyUpdated"`
+		DailySuspended []uint `json:"dailySuspended"`
+	}
 )
 
 const (