diff --git a/crm/repository/content.go b/crm/repository/content.go index b4fbddb34..230c7053d 100644 --- a/crm/repository/content.go +++ b/crm/repository/content.go @@ -7,10 +7,8 @@ import ( "strings" "time" - "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/titpetric/factory" - "gopkg.in/Masterminds/squirrel.v1" "github.com/crusttech/crust/crm/types" ) @@ -70,101 +68,17 @@ func (r *content) FindByID(id uint64) (*types.Content, error) { } func (r *content) Report(moduleID uint64, params *types.ContentReport) (results interface{}, err error) { - const jsonField = `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))` - - resolveField := func(name string) squirrel.Sqlizer { - var col squirrel.Sqlizer - switch name { - case "created_at", "updated_at": - col = squirrel.Expr(name) - default: - col = squirrel.Expr(jsonField, name) - } - - return col - } - - fieldAlias := func(col squirrel.Sqlizer, alias, fallback string) (squirrel.Sqlizer, string) { - if alias != "" { - return squirrel.Alias(col, alias), alias - } else { - return squirrel.Alias(col, fallback), fallback - } - } - - wrapInModifiers := func(col squirrel.Sqlizer, mm ...string) squirrel.Sqlizer { - for _, m := range mm { - switch strings.ToUpper(m) { - case "WEEKDAY": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%W')") - case "DATE": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m-%d')") - case "WEEK": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%u')") - case "MONTH": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m')") - case "QUARTER": - col = SqlConcatExpr("CONCAT(", "YEAR(", col, "), 'Q', ", "QUARTER(", col, ")", ")") - case "YEAR": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y')") - } - } - - return col - } - - report := squirrel. - Select(). - From("crm_content"). - Where("module_id = ?", moduleID) - - if params == nil { - return nil, errors.New("Can not generate report without parameters") - } - - var alias string - for i, m := range params.Metrics { - // @todo resolve/expend expression so we can support other functions than just COUNT(*) - col := resolveField(m.Expression) - - col = SqlConcatExpr("COUNT(", col, ")") - col, alias = fieldAlias(col, m.Alias, fmt.Sprintf("metric_%d", i)) - - report = report.Column(col) - } - - for i, d := range params.Dimensions { - col := resolveField(d.Field) - - col = wrapInModifiers(col, d.Modifiers...) - - col, alias = fieldAlias(col, d.Alias, fmt.Sprintf("dimension_%d", i)) - - report = report.Column(col) - report = report.GroupBy(alias) - report = report.OrderBy(alias) - } + crb := NewContentReportBuilder(moduleID, params) var result = make([]map[string]interface{}, 0) - if query, args, err := report.ToSql(); err != nil { + if query, args, err := crb.Build(); err != nil { return nil, errors.Wrap(err, "Can not generate report query") } else if rows, err := r.db().Query(query, args...); err != nil { - return nil, errors.Wrap(err, "Can not execute report query") + return nil, errors.Wrapf(err, "Can not execute report query (%s)", query) } else { for rows.Next() { - m := map[string]interface{}{} - sqlx.MapScan(rows, m) - - for k, v := range m { - switch cv := v.(type) { - case []uint8: - m[k] = string(cv) - } - } - - result = append(result, m) - + result = append(result, crb.Cast(rows)) } return result, nil diff --git a/crm/repository/content_report_builder.go b/crm/repository/content_report_builder.go new file mode 100644 index 000000000..5e295568d --- /dev/null +++ b/crm/repository/content_report_builder.go @@ -0,0 +1,151 @@ +package repository + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/davecgh/go-spew/spew" + "github.com/jmoiron/sqlx" + "gopkg.in/Masterminds/squirrel.v1" + + "github.com/crusttech/crust/crm/types" +) + +type ( + contentReportBuilder struct { + jsonField string + + moduleID uint64 + params *types.ContentReport + } +) + +var ( + contentReportExprMatch = regexp.MustCompile(`^\s*(\w+)\((.+)\)\s*$`) +) + +func NewContentReportBuilder(moduleID uint64, params *types.ContentReport) *contentReportBuilder { + return &contentReportBuilder{ + moduleID: moduleID, + params: params, + jsonField: `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`, + } +} + +func (b contentReportBuilder) field(name string) squirrel.Sqlizer { + switch name { + case "created_at", "updated_at": + return squirrel.Expr(name) + default: + return squirrel.Expr(b.jsonField, name) + } +} + +func (b contentReportBuilder) alias(col squirrel.Sqlizer, alias, fallback string) (squirrel.Sqlizer, string) { + if alias != "" { + return squirrel.Alias(col, alias), alias + } + + return squirrel.Alias(col, fallback), fallback +} + +func (b contentReportBuilder) wrapInModifiers(col squirrel.Sqlizer, mm ...string) squirrel.Sqlizer { + for _, m := range mm { + switch strings.ToUpper(m) { + case "WEEKDAY": + col = SqlConcatExpr("DATE_FORMAT(", col, ", '%W')") + case "DATE": + col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m-%d')") + case "WEEK": + col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%u')") + case "MONTH": + col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m')") + case "QUARTER": + col = SqlConcatExpr("CONCAT(", "YEAR(", col, "), 'Q', ", "QUARTER(", col, ")", ")") + case "YEAR": + col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y')") + } + } + + return col +} + +func (b contentReportBuilder) parseExpression(exp string) squirrel.Sqlizer { + res := contentReportExprMatch.FindStringSubmatch(exp) + if len(res) > 0 { + aggrFuncName := strings.ToUpper(res[1]) + aggrFuncArgs := b.parseExpression(res[2]) + + switch aggrFuncName { + case "COUNTD": + return SqlConcatExpr("COUNT(DISTINCT ", aggrFuncArgs, ")") + + case "SUM", "MAX", "MIN", "AVG", "STD": + return SqlConcatExpr(aggrFuncName+"(CAST(", aggrFuncArgs, " AS DECIMAL(14,2)))") + } + } else { + return b.field(exp) + } + + return nil +} + +func (b *contentReportBuilder) Build() (sql string, args []interface{}, err error) { + report := squirrel. + Select(). + Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")). + From("crm_content"). + Where("module_id = ?", b.moduleID) + + if b.params == nil { + return "", nil, errors.New("Can not generate report without parameters") + } + + spew.Dump(b.params) + for i, m := range b.params.Metrics { + col := SqlConcatExpr("CAST(", b.parseExpression(m.Expression), " AS DECIMAL(14,2))") + col, m.Alias = b.alias(col, m.Alias, fmt.Sprintf("metric_%d", i)) + + report = report.Column(col) + + b.params.Metrics[i].Alias = m.Alias // copy generated alias back + } + + for i, d := range b.params.Dimensions { + col := b.field(d.Field) + + col = b.wrapInModifiers(col, d.Modifiers...) + + col, d.Alias = b.alias(col, d.Alias, fmt.Sprintf("dimension_%d", i)) + + report = report.Column(col) + report = report.GroupBy(d.Alias) + report = report.OrderBy(d.Alias) + + b.params.Dimensions[i].Alias = d.Alias // copy generated alias back + } + + spew.Dump(b.params) + return report.ToSql() +} + +func (b contentReportBuilder) Cast(row sqlx.ColScanner) map[string]interface{} { + out := map[string]interface{}{} + sqlx.MapScan(row, out) + for k, v := range out { + switch cv := v.(type) { + case []uint8: + out[k] = string(cv) + } + } + + // Cast all metrics to float64 + for _, m := range b.params.Metrics { + out[m.Alias], _ = strconv.ParseFloat(out[m.Alias].(string), 64) + } + + return out +} diff --git a/crm/repository/content_report_builder_test.go b/crm/repository/content_report_builder_test.go new file mode 100644 index 000000000..5210231e6 --- /dev/null +++ b/crm/repository/content_report_builder_test.go @@ -0,0 +1,27 @@ +package repository + +import ( + "testing" +) + +func TestContentReportBuilder_parseExpression(t *testing.T) { + b := contentReportBuilder{jsonField: "JSONFIELD"} + + tc := []struct { + exp string + sql string + arg []interface{} + err error + }{ + {exp: "count(foo)", sql: "COUNT(JSONFIELD)", arg: []interface{}{"foo"}}, + {exp: "sum(count(foo))", sql: "SUM(COUNT(JSONFIELD))", arg: []interface{}{"foo"}}, + {exp: "sum( count( foo)) ", sql: "SUM(COUNT(JSONFIELD))", arg: []interface{}{"foo"}}, + } + + for _, c := range tc { + sql, arg, err := b.parseExpression(c.exp).ToSql() + assert(t, sql == c.sql, "Expecting expression SQL to match (%v == %v)", sql, c.sql) + assert(t, len(arg) == len(c.arg), "Expecting arguments count to match (%v == %v)", arg, c.arg) + assert(t, err == c.err, "Expecting errors to match (%v == %v)", err, c.err) + } +} diff --git a/crm/repository/main_test.go b/crm/repository/main_test.go new file mode 100644 index 000000000..a3481c25d --- /dev/null +++ b/crm/repository/main_test.go @@ -0,0 +1,12 @@ +package repository + +import ( + "testing" +) + +func assert(t *testing.T, ok bool, format string, args ...interface{}) bool { + if !ok { + t.Fatalf(format, args...) + } + return ok +} diff --git a/crm/rest/module.go b/crm/rest/module.go index 78d550a70..43f23b482 100644 --- a/crm/rest/module.go +++ b/crm/rest/module.go @@ -58,8 +58,14 @@ func (s *Module) Edit(ctx context.Context, r *request.ModuleEdit) (interface{}, func (s *Module) ContentReport(ctx context.Context, r *request.ModuleContentReport) (interface{}, error) { reportParams := &types.ContentReport{} - reportParams.ScanMetrics(strings.Split(r.Metrics, ",")...) - reportParams.ScanDimensions(strings.Split(r.Dimensions, ",")...) + if strings.TrimSpace(r.Metrics) != "" { + reportParams.ScanMetrics(strings.Split(r.Metrics, ",")...) + } + + if strings.TrimSpace(r.Dimensions) != "" { + reportParams.ScanDimensions(strings.Split(r.Dimensions, ",")...) + } + return s.content.With(ctx).Report(r.ModuleID, reportParams) } diff --git a/crm/types/content_util.go b/crm/types/content_util.go index a1c063262..42c5442f5 100644 --- a/crm/types/content_util.go +++ b/crm/types/content_util.go @@ -29,7 +29,7 @@ var ( ) func init() { - contentReportMetricScanRE = regexp.MustCompile("^(?:(\\w+):)?(\\w+)$") + contentReportMetricScanRE = regexp.MustCompile("^(?:(\\w+):)?(.+)$") contentReportDimensionScanRE = regexp.MustCompile("^(?:(\\w+):)?(\\w+)((?:\\|?\\w+)+)?$") }