DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Improve reporting, support basic metric expressions

Browse Source
This commit is contained in:
Denis Arh
2018-12-12 13:47:46 +01:00
parent 93f8eed8f3
commit d30353d4f5
6 changed files with 203 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crm/repository/content.go
+4 -90
View File
@@ -7,10 +7,8 @@ import (
"strings"
"time"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/titpetric/factory"
"gopkg.in/Masterminds/squirrel.v1"
"github.com/crusttech/crust/crm/types"
)
@@ -70,101 +68,17 @@ func (r *content) FindByID(id uint64) (*types.Content, error) {
}
func (r *content) Report(moduleID uint64, params *types.ContentReport) (results interface{}, err error) {
const jsonField = `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`
resolveField := func(name string) squirrel.Sqlizer {
var col squirrel.Sqlizer
switch name {
case "created_at", "updated_at":
col = squirrel.Expr(name)
default:
col = squirrel.Expr(jsonField, name)
}
return col
}
fieldAlias := func(col squirrel.Sqlizer, alias, fallback string) (squirrel.Sqlizer, string) {
if alias != "" {
return squirrel.Alias(col, alias), alias
} else {
return squirrel.Alias(col, fallback), fallback
}
}
wrapInModifiers := func(col squirrel.Sqlizer, mm ...string) squirrel.Sqlizer {
for _, m := range mm {
switch strings.ToUpper(m) {
case "WEEKDAY":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%W')")
case "DATE":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m-%d')")
case "WEEK":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%u')")
case "MONTH":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m')")
case "QUARTER":
col = SqlConcatExpr("CONCAT(", "YEAR(", col, "), 'Q', ", "QUARTER(", col, ")", ")")
case "YEAR":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y')")
}
}
return col
}
report := squirrel.
Select().
From("crm_content").
Where("module_id = ?", moduleID)
if params == nil {
return nil, errors.New("Can not generate report without parameters")
}
var alias string
for i, m := range params.Metrics {
// @todo resolve/expend expression so we can support other functions than just COUNT(*)
col := resolveField(m.Expression)
col = SqlConcatExpr("COUNT(", col, ")")
col, alias = fieldAlias(col, m.Alias, fmt.Sprintf("metric_%d", i))
report = report.Column(col)
}
for i, d := range params.Dimensions {
col := resolveField(d.Field)
col = wrapInModifiers(col, d.Modifiers...)
col, alias = fieldAlias(col, d.Alias, fmt.Sprintf("dimension_%d", i))
report = report.Column(col)
report = report.GroupBy(alias)
report = report.OrderBy(alias)
}
crb := NewContentReportBuilder(moduleID, params)
var result = make([]map[string]interface{}, 0)
if query, args, err := report.ToSql(); err != nil {
if query, args, err := crb.Build(); err != nil {
return nil, errors.Wrap(err, "Can not generate report query")
} else if rows, err := r.db().Query(query, args...); err != nil {
return nil, errors.Wrap(err, "Can not execute report query")
return nil, errors.Wrapf(err, "Can not execute report query (%s)", query)
} else {
for rows.Next() {
m := map[string]interface{}{}
sqlx.MapScan(rows, m)
for k, v := range m {
switch cv := v.(type) {
case []uint8:
m[k] = string(cv)
}
}
result = append(result, m)
result = append(result, crb.Cast(rows))
}
return result, nil
crm/repository/content_report_builder.go
+151
View File
@@ -0,0 +1,151 @@
package repository
import (
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/davecgh/go-spew/spew"
"github.com/jmoiron/sqlx"
"gopkg.in/Masterminds/squirrel.v1"
"github.com/crusttech/crust/crm/types"
)
type (
contentReportBuilder struct {
jsonField string
moduleID uint64
params *types.ContentReport
}
)
var (
contentReportExprMatch = regexp.MustCompile(`^\s*(\w+)\((.+)\)\s*$`)
)
func NewContentReportBuilder(moduleID uint64, params *types.ContentReport) *contentReportBuilder {
return &contentReportBuilder{
moduleID: moduleID,
params: params,
jsonField: `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`,
}
}
func (b contentReportBuilder) field(name string) squirrel.Sqlizer {
switch name {
case "created_at", "updated_at":
return squirrel.Expr(name)
default:
return squirrel.Expr(b.jsonField, name)
}
}
func (b contentReportBuilder) alias(col squirrel.Sqlizer, alias, fallback string) (squirrel.Sqlizer, string) {
if alias != "" {
return squirrel.Alias(col, alias), alias
}
return squirrel.Alias(col, fallback), fallback
}
func (b contentReportBuilder) wrapInModifiers(col squirrel.Sqlizer, mm ...string) squirrel.Sqlizer {
for _, m := range mm {
switch strings.ToUpper(m) {
case "WEEKDAY":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%W')")
case "DATE":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m-%d')")
case "WEEK":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%u')")
case "MONTH":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m')")
case "QUARTER":
col = SqlConcatExpr("CONCAT(", "YEAR(", col, "), 'Q', ", "QUARTER(", col, ")", ")")
case "YEAR":
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y')")
}
}
return col
}
func (b contentReportBuilder) parseExpression(exp string) squirrel.Sqlizer {
res := contentReportExprMatch.FindStringSubmatch(exp)
if len(res) > 0 {
aggrFuncName := strings.ToUpper(res[1])
aggrFuncArgs := b.parseExpression(res[2])
switch aggrFuncName {
case "COUNTD":
return SqlConcatExpr("COUNT(DISTINCT ", aggrFuncArgs, ")")
case "SUM", "MAX", "MIN", "AVG", "STD":
return SqlConcatExpr(aggrFuncName+"(CAST(", aggrFuncArgs, " AS DECIMAL(14,2)))")
}
} else {
return b.field(exp)
}
return nil
}
func (b *contentReportBuilder) Build() (sql string, args []interface{}, err error) {
report := squirrel.
Select().
Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")).
From("crm_content").
Where("module_id = ?", b.moduleID)
if b.params == nil {
return "", nil, errors.New("Can not generate report without parameters")
}
spew.Dump(b.params)
for i, m := range b.params.Metrics {
col := SqlConcatExpr("CAST(", b.parseExpression(m.Expression), " AS DECIMAL(14,2))")
col, m.Alias = b.alias(col, m.Alias, fmt.Sprintf("metric_%d", i))
report = report.Column(col)
b.params.Metrics[i].Alias = m.Alias // copy generated alias back
}
for i, d := range b.params.Dimensions {
col := b.field(d.Field)
col = b.wrapInModifiers(col, d.Modifiers...)
col, d.Alias = b.alias(col, d.Alias, fmt.Sprintf("dimension_%d", i))
report = report.Column(col)
report = report.GroupBy(d.Alias)
report = report.OrderBy(d.Alias)
b.params.Dimensions[i].Alias = d.Alias // copy generated alias back
}
spew.Dump(b.params)
return report.ToSql()
}
func (b contentReportBuilder) Cast(row sqlx.ColScanner) map[string]interface{} {
out := map[string]interface{}{}
sqlx.MapScan(row, out)
for k, v := range out {
switch cv := v.(type) {
case []uint8:
out[k] = string(cv)
}
}
// Cast all metrics to float64
for _, m := range b.params.Metrics {
out[m.Alias], _ = strconv.ParseFloat(out[m.Alias].(string), 64)
}
return out
}
crm/repository/content_report_builder_test.go
+27
View File
@@ -0,0 +1,27 @@
package repository
import (
"testing"
)
func TestContentReportBuilder_parseExpression(t *testing.T) {
b := contentReportBuilder{jsonField: "JSONFIELD"}
tc := []struct {
exp string
sql string
arg []interface{}
err error
}{
{exp: "count(foo)", sql: "COUNT(JSONFIELD)", arg: []interface{}{"foo"}},
{exp: "sum(count(foo))", sql: "SUM(COUNT(JSONFIELD))", arg: []interface{}{"foo"}},
{exp: "sum( count( foo)) ", sql: "SUM(COUNT(JSONFIELD))", arg: []interface{}{"foo"}},
}
for _, c := range tc {
sql, arg, err := b.parseExpression(c.exp).ToSql()
assert(t, sql == c.sql, "Expecting expression SQL to match (%v == %v)", sql, c.sql)
assert(t, len(arg) == len(c.arg), "Expecting arguments count to match (%v == %v)", arg, c.arg)
assert(t, err == c.err, "Expecting errors to match (%v == %v)", err, c.err)
}
}
crm/repository/main_test.go
+12
View File
@@ -0,0 +1,12 @@
package repository
import (
"testing"
)
func assert(t *testing.T, ok bool, format string, args ...interface{}) bool {
if !ok {
t.Fatalf(format, args...)
}
return ok
}
crm/rest/module.go
+8 -2
View File
@@ -58,8 +58,14 @@ func (s *Module) Edit(ctx context.Context, r *request.ModuleEdit) (interface{},
func (s *Module) ContentReport(ctx context.Context, r *request.ModuleContentReport) (interface{}, error) {
reportParams := &types.ContentReport{}
reportParams.ScanMetrics(strings.Split(r.Metrics, ",")...)
reportParams.ScanDimensions(strings.Split(r.Dimensions, ",")...)
if strings.TrimSpace(r.Metrics) != "" {
reportParams.ScanMetrics(strings.Split(r.Metrics, ",")...)
}
if strings.TrimSpace(r.Dimensions) != "" {
reportParams.ScanDimensions(strings.Split(r.Dimensions, ",")...)
}
return s.content.With(ctx).Report(r.ModuleID, reportParams)
}
crm/types/content_util.go
+1 -1
View File
@@ -29,7 +29,7 @@ var (
)
func init() {
contentReportMetricScanRE = regexp.MustCompile("^(?:(\\w+):)?(\\w+)$")
contentReportMetricScanRE = regexp.MustCompile("^(?:(\\w+):)?(.+)$")
contentReportDimensionScanRE = regexp.MustCompile("^(?:(\\w+):)?(\\w+)((?:\\|?\\w+)+)?$")
}