DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Fix metrics for apps, roles & users

Browse Source
This commit is contained in:
Denis Arh
2020-11-10 09:43:13 +01:00
parent ba753c068f
commit ee67a78f3d
9 changed files with 159 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
store/mysql/mysql.go
View File

@@ -49,6 +49,8 @@ func Connect(ctx context.Context, dsn string) (store.Storer, error) {
return nil, err
}
// See https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html#sqlmode_ansi
// for details
s.DB().ExecContext(ctx, "SET SESSION sql_mode = 'ANSI'")
return s, nil

4
store/rdbms/applications.go
View File

@@ -35,8 +35,8 @@ func (s Store) ApplicationMetrics(ctx context.Context) (*types.ApplicationMetric
counters = squirrel.
Select(
"COUNT(*) as total",
"SUM(IF(deleted_at IS NULL, 0, 1)) as deleted",
"SUM(IF(deleted_at IS NULL, 1, 0)) as valid",
"SUM(CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END) as deleted",
"SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as valid",
).
PlaceholderFormat(s.config.PlaceholderFormat).
From(s.applicationTable("u"))

62
store/rdbms/metrics.go
View File

@@ -2,8 +2,11 @@ package rdbms
import (
"context"
"database/sql"
"fmt"
"github.com/Masterminds/squirrel"
"github.com/cortezaproject/corteza-server/pkg/ql"
"time"
)
// multiDailyMetrics simplifies fetching of multiple daily metrics
@@ -28,32 +31,59 @@ func (s Store) multiDailyMetrics(ctx context.Context, q squirrel.SelectBuilder,
// This function is copied from old repository and adapted to work under store,
// might be good to refactor it to fit into this store pattern a bit more
func (s Store) dailyMetrics(ctx context.Context, q squirrel.SelectBuilder, field string) ([]uint, error) {
q = q.
Column(fmt.Sprintf("UNIX_TIMESTAMP(DATE(%s)) timestamp", field)).
Column("COUNT(*) AS value").
Where(fmt.Sprintf("%s IS NOT NULL", field)).
OrderBy("timestamp").
GroupBy("timestamp")
var (
rval = make([]uint, 0, 100)
ts, v, i uint
rows, err = s.Query(ctx, q)
)
sqlfn, err := s.SqlFunctionHandler(ql.Function{Name: "DATE", Arguments: ql.ASTSet{ql.Ident{Value: field}}})
if err != nil {
return nil, err
}
q = q.
Column(sqlfn).
Column("COUNT(*) AS value").
Where(fmt.Sprintf("%s IS NOT NULL", field))
// since orderBy & groupBy do not accept Sqlizer, we'll convert it up front
// args are omitted because there should be none
// error is checked when adding sqlfn to Column fn
sqlfnStr, _, _ := sqlfn.ToSql()
q = q.
OrderBy(sqlfnStr).
GroupBy(sqlfnStr)
const (
cap = 100
)
var (
rval = make([]uint, 0, cap)
d time.Time
ts string
c uint
rows *sql.Rows
)
return rval, func() (err error) {
rows, err = s.Query(ctx, q)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
if err = rows.Scan(&ts, &v); err != nil {
if err = rows.Scan(&ts, &c); err != nil {
return err
}
rval[2*i], rval[2*i+1] = ts, v
i++
d = time.Time{}
if ts != "" {
if d, err = time.Parse(time.RFC3339, ts); err != nil {
return err
}
}
rval = append(rval, uint(d.Unix()), c)
}
return

8
store/rdbms/roles.go
View File

@@ -49,9 +49,9 @@ func (s Store) RoleMetrics(ctx context.Context) (*types.RoleMetrics, error) {
counters = squirrel.
Select(
"COUNT(*) as total",
"SUM(IF(deleted_at IS NULL, 0, 1)) as deleted",
"SUM(IF(archived_at IS NULL, 0, 1)) as archived",
"SUM(IF(deleted_at IS NULL AND archived_at IS NULL, 1, 0)) as valid",
"SUM(CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END) as deleted",
"SUM(CASE WHEN archived_at IS NULL THEN 0 ELSE 1 END) as archived",
"SUM(CASE WHEN deleted_at IS NULL AND archived_at IS NULL THEN 1 ELSE 0 END) as valid",
).
PlaceholderFormat(s.config.PlaceholderFormat).
From(s.roleTable("u"))
@@ -80,7 +80,7 @@ func (s Store) RoleMetrics(ctx context.Context) (*types.RoleMetrics, error) {
"created_at",
"updated_at",
"deleted_at",
"suspended_at",
"archived_at",
},
&rval.DailyCreated,
&rval.DailyUpdated,

6
store/rdbms/users.go
View File

@@ -77,9 +77,9 @@ func (s Store) UserMetrics(ctx context.Context) (*types.UserMetrics, error) {
counters = squirrel.
Select(
"COUNT(*) as total",
"SUM(IF(deleted_at IS NULL, 0, 1)) as deleted",
"SUM(IF(suspended_at IS NULL, 0, 1)) as suspended",
"SUM(IF(deleted_at IS NULL AND suspended_at IS NULL, 1, 0)) as valid",
"SUM(CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END) as deleted",
"SUM(CASE WHEN suspended_at IS NULL THEN 0 ELSE 1 END) as suspended",
"SUM(CASE WHEN deleted_at IS NULL AND suspended_at IS NULL THEN 1 ELSE 0 END) as valid",
).
PlaceholderFormat(s.config.PlaceholderFormat).
From(s.userTable("u"))

4
store/sqlite3/sql_functions.go
View File

@@ -18,8 +18,10 @@ func sqlFunctionHandler(f ql.Function) (ql.ASTNode, error) {
if len(f.Arguments) != 2 {
return nil, fmt.Errorf("expecting exactly two arguments for DATE_FORMAT function")
}
return ql.MakeFormattedNode("STRFTIME('%s', %s)", f.Arguments[0], f.Arguments[1]), nil
case "DATE":
// need to convert back to datetime so it can be converted to time.Time
return ql.MakeFormattedNode("STRFTIME('%%Y-%%m-%%dT00:00:00Z', %s)", f.Arguments...), nil
case "DATE_ADD", "DATE_SUB", "STD":
return nil, fmt.Errorf("%q function is currently unsupported in SQLite store backend", f.Name)
}

29
store/tests/applications_test.go
View File

@@ -67,9 +67,9 @@ func testApplications(t *testing.T, s store.Applications) {
t.Run("existing", func(t *testing.T) {
req, application := truncAndCreate(t)
application.Name = "ApplicationCRUD+2"
req.NoError(s.UpsertApplication(ctx, application))
updated, err := s.LookupApplicationByID(ctx, application.ID)
req.NoError(err)
req.Equal(application.Name, updated.Name)
@@ -80,7 +80,7 @@ func testApplications(t *testing.T, s store.Applications) {
application.Name = "ComposeChartCRUD+2"
req.NoError(s.UpsertApplication(ctx, application))
upserted, err := s.LookupApplicationByID(ctx, application.ID)
req.NoError(err)
req.Equal(application.Name, upserted.Name)
@@ -146,4 +146,27 @@ func testApplications(t *testing.T, s store.Applications) {
_ = f // dummy
})
t.Run("metrics", func(t *testing.T) {
var (
req = require.New(t)
e = &types.ApplicationMetrics{
Total: 5,
Valid: 2,
Deleted: 3,
}
)
req.NoError(s.TruncateApplications(ctx))
req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), UpdatedAt: now()}))
req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), UpdatedAt: now()}))
req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), DeletedAt: now()}))
req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), DeletedAt: now()}))
req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), DeletedAt: now()}))
m, err := store.ApplicationMetrics(ctx, s)
req.NoError(err)
req.Equal(e, m)
})
}

37
store/tests/roles_test.go
View File

@@ -155,4 +155,41 @@ func testRoles(t *testing.T, s store.Roles) {
t.Run("ordered search", func(t *testing.T) {
t.Skip("not implemented")
})
t.Run("metrics", func(t *testing.T) {
var (
req = require.New(t)
oct, _ = time.Parse(time.RFC3339, "2020-10-02T10:01:10Z")
nov, _ = time.Parse(time.RFC3339, "2020-11-02T20:02:10Z")
octu = uint(oct.Truncate(time.Hour * 24).Unix())
novu = uint(nov.Truncate(time.Hour * 24).Unix())
e = &types.RoleMetrics{
Total: 8,
Valid: 2,
Deleted: 3,
Archived: 3,
DailyCreated: []uint{octu, 7, novu, 1},
DailyUpdated: []uint{octu, 2},
DailyArchived: []uint{octu, 2, novu, 1},
DailyDeleted: []uint{novu, 3},
}
)
req.NoError(s.TruncateRoles(ctx))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, UpdatedAt: &oct}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, UpdatedAt: &oct}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, ArchivedAt: &oct}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, ArchivedAt: &oct}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, ArchivedAt: &nov}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, DeletedAt: &nov}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, DeletedAt: &nov}))
req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: nov, DeletedAt: &nov}))
m, err := store.RoleMetrics(ctx, s)
req.NoError(err)
req.Equal(e, m)
})
}

36
store/tests/users_test.go
View File

@@ -345,4 +345,40 @@ func testUsers(t *testing.T, s store.Users) {
req.Equal(c1, c2)
})
t.Run("metrics", func(t *testing.T) {
var (
req = require.New(t)
oct, _ = time.Parse(time.RFC3339, "2020-10-02T10:01:10Z")
nov, _ = time.Parse(time.RFC3339, "2020-11-02T20:02:10Z")
octu = uint(oct.Truncate(time.Hour * 24).Unix())
novu = uint(nov.Truncate(time.Hour * 24).Unix())
e = &types.UserMetrics{
Total: 8,
Valid: 2,
Deleted: 3,
Suspended: 3,
DailyCreated: []uint{octu, 7, novu, 1},
DailyUpdated: []uint{octu, 2},
DailySuspended: []uint{octu, 2, novu, 1},
DailyDeleted: []uint{novu, 3},
}
)
req.NoError(s.TruncateUsers(ctx))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), UpdatedAt: &oct}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), UpdatedAt: &oct}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), SuspendedAt: &oct}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), SuspendedAt: &oct}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), SuspendedAt: &nov}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), DeletedAt: &nov}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), DeletedAt: &nov}))
req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: nov, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), DeletedAt: &nov}))
m, err := store.UserMetrics(ctx, s)
req.NoError(err)
req.Equal(e, m)
})
}