From ee67a78f3d92ad5b1bbd993fc98af4c59bb7e390 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Tue, 10 Nov 2020 09:43:13 +0100 Subject: [PATCH] Fix metrics for apps, roles & users --- store/mysql/mysql.go | 2 ++ store/rdbms/applications.go | 4 +-- store/rdbms/metrics.go | 62 +++++++++++++++++++++++--------- store/rdbms/roles.go | 8 ++--- store/rdbms/users.go | 6 ++-- store/sqlite3/sql_functions.go | 4 ++- store/tests/applications_test.go | 29 +++++++++++++-- store/tests/roles_test.go | 37 +++++++++++++++++++ store/tests/users_test.go | 36 +++++++++++++++++++ 9 files changed, 159 insertions(+), 29 deletions(-) diff --git a/store/mysql/mysql.go b/store/mysql/mysql.go index fc5fe6cff..f617bef18 100644 --- a/store/mysql/mysql.go +++ b/store/mysql/mysql.go @@ -49,6 +49,8 @@ func Connect(ctx context.Context, dsn string) (store.Storer, error) { return nil, err } + // See https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html#sqlmode_ansi + // for details s.DB().ExecContext(ctx, "SET SESSION sql_mode = 'ANSI'") return s, nil diff --git a/store/rdbms/applications.go b/store/rdbms/applications.go index ca7d121d4..08501c10f 100644 --- a/store/rdbms/applications.go +++ b/store/rdbms/applications.go @@ -35,8 +35,8 @@ func (s Store) ApplicationMetrics(ctx context.Context) (*types.ApplicationMetric counters = squirrel. Select( "COUNT(*) as total", - "SUM(IF(deleted_at IS NULL, 0, 1)) as deleted", - "SUM(IF(deleted_at IS NULL, 1, 0)) as valid", + "SUM(CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END) as deleted", + "SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as valid", ). PlaceholderFormat(s.config.PlaceholderFormat). From(s.applicationTable("u")) diff --git a/store/rdbms/metrics.go b/store/rdbms/metrics.go index e81d5788e..9c742ef16 100644 --- a/store/rdbms/metrics.go +++ b/store/rdbms/metrics.go @@ -2,8 +2,11 @@ package rdbms import ( "context" + "database/sql" "fmt" "github.com/Masterminds/squirrel" + "github.com/cortezaproject/corteza-server/pkg/ql" + "time" ) // multiDailyMetrics simplifies fetching of multiple daily metrics @@ -28,32 +31,59 @@ func (s Store) multiDailyMetrics(ctx context.Context, q squirrel.SelectBuilder, // This function is copied from old repository and adapted to work under store, // might be good to refactor it to fit into this store pattern a bit more func (s Store) dailyMetrics(ctx context.Context, q squirrel.SelectBuilder, field string) ([]uint, error) { - q = q. - Column(fmt.Sprintf("UNIX_TIMESTAMP(DATE(%s)) timestamp", field)). - Column("COUNT(*) AS value"). - Where(fmt.Sprintf("%s IS NOT NULL", field)). - OrderBy("timestamp"). - GroupBy("timestamp") - - var ( - rval = make([]uint, 0, 100) - ts, v, i uint - rows, err = s.Query(ctx, q) - ) - + sqlfn, err := s.SqlFunctionHandler(ql.Function{Name: "DATE", Arguments: ql.ASTSet{ql.Ident{Value: field}}}) if err != nil { return nil, err } + q = q. + Column(sqlfn). + Column("COUNT(*) AS value"). + Where(fmt.Sprintf("%s IS NOT NULL", field)) + + // since orderBy & groupBy do not accept Sqlizer, we'll convert it up front + // args are omitted because there should be none + // error is checked when adding sqlfn to Column fn + sqlfnStr, _, _ := sqlfn.ToSql() + + q = q. + OrderBy(sqlfnStr). + GroupBy(sqlfnStr) + + const ( + cap = 100 + ) + + var ( + rval = make([]uint, 0, cap) + + d time.Time + ts string + c uint + + rows *sql.Rows + ) + return rval, func() (err error) { + rows, err = s.Query(ctx, q) + if err != nil { + return err + } + defer rows.Close() for rows.Next() { - if err = rows.Scan(&ts, &v); err != nil { + if err = rows.Scan(&ts, &c); err != nil { return err } - rval[2*i], rval[2*i+1] = ts, v - i++ + d = time.Time{} + if ts != "" { + if d, err = time.Parse(time.RFC3339, ts); err != nil { + return err + } + } + + rval = append(rval, uint(d.Unix()), c) } return diff --git a/store/rdbms/roles.go b/store/rdbms/roles.go index d79ec4ce0..2283a61fe 100644 --- a/store/rdbms/roles.go +++ b/store/rdbms/roles.go @@ -49,9 +49,9 @@ func (s Store) RoleMetrics(ctx context.Context) (*types.RoleMetrics, error) { counters = squirrel. Select( "COUNT(*) as total", - "SUM(IF(deleted_at IS NULL, 0, 1)) as deleted", - "SUM(IF(archived_at IS NULL, 0, 1)) as archived", - "SUM(IF(deleted_at IS NULL AND archived_at IS NULL, 1, 0)) as valid", + "SUM(CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END) as deleted", + "SUM(CASE WHEN archived_at IS NULL THEN 0 ELSE 1 END) as archived", + "SUM(CASE WHEN deleted_at IS NULL AND archived_at IS NULL THEN 1 ELSE 0 END) as valid", ). PlaceholderFormat(s.config.PlaceholderFormat). From(s.roleTable("u")) @@ -80,7 +80,7 @@ func (s Store) RoleMetrics(ctx context.Context) (*types.RoleMetrics, error) { "created_at", "updated_at", "deleted_at", - "suspended_at", + "archived_at", }, &rval.DailyCreated, &rval.DailyUpdated, diff --git a/store/rdbms/users.go b/store/rdbms/users.go index 39fff6991..53862e37d 100644 --- a/store/rdbms/users.go +++ b/store/rdbms/users.go @@ -77,9 +77,9 @@ func (s Store) UserMetrics(ctx context.Context) (*types.UserMetrics, error) { counters = squirrel. Select( "COUNT(*) as total", - "SUM(IF(deleted_at IS NULL, 0, 1)) as deleted", - "SUM(IF(suspended_at IS NULL, 0, 1)) as suspended", - "SUM(IF(deleted_at IS NULL AND suspended_at IS NULL, 1, 0)) as valid", + "SUM(CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END) as deleted", + "SUM(CASE WHEN suspended_at IS NULL THEN 0 ELSE 1 END) as suspended", + "SUM(CASE WHEN deleted_at IS NULL AND suspended_at IS NULL THEN 1 ELSE 0 END) as valid", ). PlaceholderFormat(s.config.PlaceholderFormat). From(s.userTable("u")) diff --git a/store/sqlite3/sql_functions.go b/store/sqlite3/sql_functions.go index dfb6deac2..3807b2914 100644 --- a/store/sqlite3/sql_functions.go +++ b/store/sqlite3/sql_functions.go @@ -18,8 +18,10 @@ func sqlFunctionHandler(f ql.Function) (ql.ASTNode, error) { if len(f.Arguments) != 2 { return nil, fmt.Errorf("expecting exactly two arguments for DATE_FORMAT function") } - return ql.MakeFormattedNode("STRFTIME('%s', %s)", f.Arguments[0], f.Arguments[1]), nil + case "DATE": + // need to convert back to datetime so it can be converted to time.Time + return ql.MakeFormattedNode("STRFTIME('%%Y-%%m-%%dT00:00:00Z', %s)", f.Arguments...), nil case "DATE_ADD", "DATE_SUB", "STD": return nil, fmt.Errorf("%q function is currently unsupported in SQLite store backend", f.Name) } diff --git a/store/tests/applications_test.go b/store/tests/applications_test.go index c5cd955a7..cb83b2b27 100644 --- a/store/tests/applications_test.go +++ b/store/tests/applications_test.go @@ -67,9 +67,9 @@ func testApplications(t *testing.T, s store.Applications) { t.Run("existing", func(t *testing.T) { req, application := truncAndCreate(t) application.Name = "ApplicationCRUD+2" - + req.NoError(s.UpsertApplication(ctx, application)) - + updated, err := s.LookupApplicationByID(ctx, application.ID) req.NoError(err) req.Equal(application.Name, updated.Name) @@ -80,7 +80,7 @@ func testApplications(t *testing.T, s store.Applications) { application.Name = "ComposeChartCRUD+2" req.NoError(s.UpsertApplication(ctx, application)) - + upserted, err := s.LookupApplicationByID(ctx, application.ID) req.NoError(err) req.Equal(application.Name, upserted.Name) @@ -146,4 +146,27 @@ func testApplications(t *testing.T, s store.Applications) { _ = f // dummy }) + + t.Run("metrics", func(t *testing.T) { + var ( + req = require.New(t) + + e = &types.ApplicationMetrics{ + Total: 5, + Valid: 2, + Deleted: 3, + } + ) + + req.NoError(s.TruncateApplications(ctx)) + req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), UpdatedAt: now()})) + req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), UpdatedAt: now()})) + req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), DeletedAt: now()})) + req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), DeletedAt: now()})) + req.NoError(s.CreateApplication(ctx, &types.Application{ID: id.Next(), CreatedAt: *now(), DeletedAt: now()})) + + m, err := store.ApplicationMetrics(ctx, s) + req.NoError(err) + req.Equal(e, m) + }) } diff --git a/store/tests/roles_test.go b/store/tests/roles_test.go index 013f5644e..52076a4f6 100644 --- a/store/tests/roles_test.go +++ b/store/tests/roles_test.go @@ -155,4 +155,41 @@ func testRoles(t *testing.T, s store.Roles) { t.Run("ordered search", func(t *testing.T) { t.Skip("not implemented") }) + + t.Run("metrics", func(t *testing.T) { + var ( + req = require.New(t) + + oct, _ = time.Parse(time.RFC3339, "2020-10-02T10:01:10Z") + nov, _ = time.Parse(time.RFC3339, "2020-11-02T20:02:10Z") + + octu = uint(oct.Truncate(time.Hour * 24).Unix()) + novu = uint(nov.Truncate(time.Hour * 24).Unix()) + + e = &types.RoleMetrics{ + Total: 8, + Valid: 2, + Deleted: 3, + Archived: 3, + DailyCreated: []uint{octu, 7, novu, 1}, + DailyUpdated: []uint{octu, 2}, + DailyArchived: []uint{octu, 2, novu, 1}, + DailyDeleted: []uint{novu, 3}, + } + ) + + req.NoError(s.TruncateRoles(ctx)) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, UpdatedAt: &oct})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, UpdatedAt: &oct})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, ArchivedAt: &oct})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, ArchivedAt: &oct})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, ArchivedAt: &nov})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, DeletedAt: &nov})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: oct, DeletedAt: &nov})) + req.NoError(s.CreateRole(ctx, &types.Role{ID: id.Next(), CreatedAt: nov, DeletedAt: &nov})) + + m, err := store.RoleMetrics(ctx, s) + req.NoError(err) + req.Equal(e, m) + }) } diff --git a/store/tests/users_test.go b/store/tests/users_test.go index dcce11783..ae1136764 100644 --- a/store/tests/users_test.go +++ b/store/tests/users_test.go @@ -345,4 +345,40 @@ func testUsers(t *testing.T, s store.Users) { req.Equal(c1, c2) }) + t.Run("metrics", func(t *testing.T) { + var ( + req = require.New(t) + + oct, _ = time.Parse(time.RFC3339, "2020-10-02T10:01:10Z") + nov, _ = time.Parse(time.RFC3339, "2020-11-02T20:02:10Z") + + octu = uint(oct.Truncate(time.Hour * 24).Unix()) + novu = uint(nov.Truncate(time.Hour * 24).Unix()) + + e = &types.UserMetrics{ + Total: 8, + Valid: 2, + Deleted: 3, + Suspended: 3, + DailyCreated: []uint{octu, 7, novu, 1}, + DailyUpdated: []uint{octu, 2}, + DailySuspended: []uint{octu, 2, novu, 1}, + DailyDeleted: []uint{novu, 3}, + } + ) + + req.NoError(s.TruncateUsers(ctx)) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), UpdatedAt: &oct})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), UpdatedAt: &oct})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), SuspendedAt: &oct})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), SuspendedAt: &oct})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), SuspendedAt: &nov})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), DeletedAt: &nov})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: oct, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), DeletedAt: &nov})) + req.NoError(s.CreateUser(ctx, &types.User{ID: id.Next(), CreatedAt: nov, Email: fmt.Sprintf("user-crud+%s@crust.test", rand.Bytes(10)), DeletedAt: &nov})) + + m, err := store.UserMetrics(ctx, s) + req.NoError(err) + req.Equal(e, m) + }) }