Improve reporting metrics/dimensions/filtering capabilities
Add lexer & ast tree parser to assist with record querying
This commit is contained in:
Denis Arh
+8
-2
@@ -676,13 +676,19 @@
|
||||
"type": "string",
|
||||
"name": "metrics",
|
||||
"required": true,
|
||||
"title": "Metrics (syntax: alias:expression;...)"
|
||||
"title": "Metrics (eg: 'COUNT(*) AS count, SUM(money)')"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"name": "dimensions",
|
||||
"required": true,
|
||||
"title": "Dimensions (syntax: alias:field|modifier|modifier2;...)"
|
||||
"title": "Dimensions (eg: 'DATE(foo), status')"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"name": "filter",
|
||||
"required": true,
|
||||
"title": "Filter (eg: 'DATE(foo) > 2010')"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -126,13 +126,19 @@
|
||||
{
|
||||
"name": "metrics",
|
||||
"required": true,
|
||||
"title": "Metrics (syntax: alias:expression;...)",
|
||||
"title": "Metrics (eg: 'COUNT(*) AS count, SUM(money)')",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "dimensions",
|
||||
"required": true,
|
||||
"title": "Dimensions (syntax: alias:field|modifier|modifier2;...)",
|
||||
"title": "Dimensions (eg: 'DATE(foo), status')",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "filter",
|
||||
"required": true,
|
||||
"title": "Filter (eg: 'DATE(foo) \u003e 2010')",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# QueryLanguage (ql) package
|
||||
|
||||
_This package was written mainly to assist with column
|
||||
conversion (wrapping non-physical columns into functions that
|
||||
extract values from a JSON value)_
|
||||
|
||||
Provides lexer and ast parser to convert a set of simple
|
||||
instructions (see `ast_parser_test.go` for examples) that closely
|
||||
resemble SQL into syntax trees.
|
||||
|
||||
It provides handlers for identifiers and functions that can
|
||||
validate and modify nodes and tree at parse time.
|
||||
|
||||
Tree can than be converted back to SQL or to structs that
|
||||
assist Squirrel select builder.
|
||||
|
||||
## Pending improvements
|
||||
|
||||
### Lexer
|
||||
|
||||
- Operator validation
|
||||
|
||||
### AST Parser
|
||||
|
||||
- simplify / combine ASTNode vs ASTSet vs Columns
|
||||
- improve resilience and detect basic syntax errors
|
||||
- properly handling COUNT(DISTINCT ...) ident syntax
|
||||
- parsing complex expressions (eg: `year(created_at) > year(NOW()) - 2`)
|
||||
@@ -0,0 +1,102 @@
|
||||
package ql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gopkg.in/Masterminds/squirrel.v1"
|
||||
)
|
||||
|
||||
// SelectStatement represents a SQL SELECT statement.
|
||||
type (
|
||||
ASTNode interface {
|
||||
fmt.Stringer
|
||||
squirrel.Sqlizer
|
||||
}
|
||||
|
||||
ASTSet []ASTNode // Stream of comma delimited nodes
|
||||
ASTNodes []ASTNode // Stream of un-delimited nodes
|
||||
|
||||
Columns []Column
|
||||
|
||||
String struct {
|
||||
Value string
|
||||
Args []interface{}
|
||||
}
|
||||
|
||||
Number struct {
|
||||
Value string
|
||||
}
|
||||
|
||||
Operator struct {
|
||||
Kind string
|
||||
}
|
||||
|
||||
Ident struct {
|
||||
Value string
|
||||
Args []interface{}
|
||||
}
|
||||
|
||||
Keyword struct {
|
||||
Keyword string
|
||||
}
|
||||
|
||||
Column struct {
|
||||
Expr ASTNodes
|
||||
Alias string
|
||||
}
|
||||
Function struct {
|
||||
Name string
|
||||
Arguments ASTSet
|
||||
}
|
||||
)
|
||||
|
||||
func (n String) String() string { return fmt.Sprintf("%q", n.Value) }
|
||||
|
||||
func (n Number) String() string { return n.Value }
|
||||
|
||||
func (n Operator) String() string { return n.Kind }
|
||||
|
||||
func (n Keyword) String() string { return n.Keyword }
|
||||
|
||||
func (n Function) String() string { return fmt.Sprintf("%s(%s)", n.Name, n.Arguments) }
|
||||
|
||||
func (n Ident) String() string { return n.Value }
|
||||
|
||||
func (n Column) String() (out string) {
|
||||
out = n.Expr.String()
|
||||
if n.Alias != "" {
|
||||
out = out + " AS " + n.Alias
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (nn ASTNodes) String() (out string) {
|
||||
for _, n := range nn {
|
||||
out = out + n.String()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (nn ASTSet) String() (out string) {
|
||||
for i, n := range nn {
|
||||
if i > 0 {
|
||||
out = out + ", "
|
||||
}
|
||||
out = out + n.String()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (nn Columns) String() (out string) {
|
||||
for i, n := range nn {
|
||||
if i > 0 {
|
||||
out = out + ", "
|
||||
}
|
||||
out = out + n.String()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
package ql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type (
|
||||
// Parser represents a parser.
|
||||
Parser struct {
|
||||
lexer *Lexer
|
||||
tokbuf []Token
|
||||
|
||||
OnIdent IdentHandler
|
||||
OnFunction FunctionHandler
|
||||
|
||||
// parenthesis level control
|
||||
level uint
|
||||
}
|
||||
|
||||
IdentHandler func(ident Ident) (Ident, error)
|
||||
FunctionHandler func(ident Function) (Function, error)
|
||||
)
|
||||
|
||||
// NewParser returns a new instance of Parser.
|
||||
func NewParser() *Parser {
|
||||
p := &Parser{
|
||||
tokbuf: make([]Token, 3),
|
||||
OnIdent: func(ident Ident) (Ident, error) { return ident, nil },
|
||||
OnFunction: func(ident Function) (Function, error) { return ident, nil },
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// Removes oldest token in the buffer, adds new one and returns 2nd oldest
|
||||
func (p *Parser) nextToken() Token {
|
||||
var t Token
|
||||
for {
|
||||
t = p.lexer.Scan()
|
||||
if !t.Is(WS) {
|
||||
p.tokbuf = append(p.tokbuf[1:], t)
|
||||
return p.tokbuf[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) peekToken(s int) Token {
|
||||
return p.tokbuf[s]
|
||||
}
|
||||
|
||||
func (p *Parser) initLexer(s string) {
|
||||
p.lexer = NewLexer(strings.NewReader(s))
|
||||
|
||||
for c := 1; c < cap(p.tokbuf); c++ {
|
||||
// Fill the buffer
|
||||
p.nextToken()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) ParseSet(s string) (ASTNode, error) {
|
||||
p.initLexer(s)
|
||||
|
||||
if set, err := p.parseSet(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return set, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) ParseExpression(s string) (ASTNode, error) {
|
||||
p.initLexer(s)
|
||||
|
||||
if set, err := p.parseExpr(p.nextToken()); err != nil {
|
||||
return nil, err
|
||||
} else if len(set) == 1 {
|
||||
return set[0], nil
|
||||
} else {
|
||||
return set, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) ParseColumns(s string) (columns Columns, err error) {
|
||||
p.initLexer(s)
|
||||
|
||||
var t Token
|
||||
var c Column
|
||||
|
||||
next:
|
||||
t = p.nextToken()
|
||||
switch t.code {
|
||||
case COMMA:
|
||||
goto next
|
||||
case EOF:
|
||||
break
|
||||
case ILLEGAL:
|
||||
return nil, fmt.Errorf("found an illegal token (%+v)", t)
|
||||
default:
|
||||
if c, err = p.parseColumn(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
columns = append(columns, c)
|
||||
}
|
||||
goto next
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (p *Parser) parseColumn(t Token) (c Column, err error) {
|
||||
if c.Expr, err = p.parseExpr(t); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Set alias move forward for 2 places
|
||||
if p.peekIfAlias() {
|
||||
c.Alias = p.peekToken(2).literal
|
||||
p.nextToken()
|
||||
p.nextToken()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Peek ahead if there is an alias ident (<IDENT:AS>
|
||||
func (p *Parser) peekIfAlias() bool {
|
||||
var f, s = p.peekToken(1), p.peekToken(2)
|
||||
return f.Is(IDENT) && strings.ToUpper(f.literal) == "AS" && s.Is(IDENT)
|
||||
}
|
||||
|
||||
func (p *Parser) parseExpr(t Token) (list ASTNodes, err error) {
|
||||
goto checkToken
|
||||
|
||||
next:
|
||||
if p.peekToken(1).Is(COMMA) || p.peekIfAlias() {
|
||||
// Peek ahead and exit on comma
|
||||
return
|
||||
}
|
||||
if p.peekToken(1).Is(PARENTHESIS_CLOSE) {
|
||||
// Peek ahead and exit on closed parenthesis
|
||||
if p.level == 0 {
|
||||
return nil, fmt.Errorf("closing unopened parenthesis in expression")
|
||||
}
|
||||
p.level--
|
||||
return
|
||||
}
|
||||
|
||||
t = p.nextToken()
|
||||
|
||||
checkToken:
|
||||
switch t.code {
|
||||
case EOF:
|
||||
break
|
||||
case WS:
|
||||
// Ignore ws... next token...
|
||||
goto next
|
||||
case ILLEGAL:
|
||||
return nil, fmt.Errorf("found an illegal token (%+v)", t)
|
||||
case IDENT:
|
||||
var ident ASTNode
|
||||
if ident, err = p.parseIdent(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, ident)
|
||||
|
||||
goto next
|
||||
}
|
||||
case OPERATOR:
|
||||
list = append(list, Operator{Kind: t.literal})
|
||||
goto next
|
||||
case NUMBER:
|
||||
list = append(list, Number{Value: t.literal})
|
||||
goto next
|
||||
case STRING:
|
||||
list = append(list, String{Value: t.literal})
|
||||
goto next
|
||||
case PARENTHESIS_OPEN:
|
||||
p.level++
|
||||
if sub, err := p.parseExpr(p.nextToken()); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, sub)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected token while parsing expression (%v)", t)
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (p *Parser) parseIdent(t Token) (list ASTNode, err error) {
|
||||
var ucLiteral = strings.ToUpper(t.literal)
|
||||
|
||||
if ucLiteral == "AND" || ucLiteral == "OR" {
|
||||
return Keyword{ucLiteral}, nil
|
||||
} else if p.peekToken(1).Is(PARENTHESIS_OPEN) {
|
||||
// Handle function calls: <IDENT><PARENTHESIS_OPEN>...
|
||||
f := Function{Name: t.literal}
|
||||
if f.Arguments, err = p.parseSet(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return p.OnFunction(f)
|
||||
}
|
||||
}
|
||||
|
||||
return p.OnIdent(Ident{Value: t.literal})
|
||||
}
|
||||
|
||||
func (p *Parser) parseSet() (list ASTSet, err error) {
|
||||
var expr ASTNodes
|
||||
var parenthesisOpened = false
|
||||
|
||||
next:
|
||||
t := p.nextToken()
|
||||
|
||||
if p.peekIfAlias() {
|
||||
return
|
||||
}
|
||||
|
||||
switch t.code {
|
||||
case WS:
|
||||
goto next
|
||||
case PARENTHESIS_OPEN:
|
||||
p.level++
|
||||
parenthesisOpened = true
|
||||
goto next
|
||||
case NUMBER:
|
||||
list = append(list, Number{Value: t.literal})
|
||||
goto next
|
||||
case STRING:
|
||||
list = append(list, String{Value: t.literal})
|
||||
goto next
|
||||
case OPERATOR:
|
||||
list = append(list, Operator{Kind: t.literal})
|
||||
goto next
|
||||
case IDENT:
|
||||
if p.peekToken(1).Is(OPERATOR) {
|
||||
// Looks like we have an expression ahead of us
|
||||
|
||||
if parenthesisOpened {
|
||||
// Expression will find closing parenthesis and dec. the level
|
||||
// so, lets bump up the number
|
||||
p.level++
|
||||
}
|
||||
|
||||
if expr, err = p.parseExpr(t); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
list = append(list, expr)
|
||||
goto next
|
||||
}
|
||||
|
||||
var ident ASTNode
|
||||
if ident, err = p.parseIdent(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, ident)
|
||||
|
||||
goto next
|
||||
}
|
||||
case COMMA:
|
||||
goto next
|
||||
case EOF:
|
||||
return
|
||||
case PARENTHESIS_CLOSE:
|
||||
// Peek ahead and exit on closed parenthesis
|
||||
if p.level == 0 {
|
||||
return nil, fmt.Errorf("closing unopened parenthesis in set")
|
||||
}
|
||||
p.level--
|
||||
return
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected token while parsing set (%v)", t)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package ql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Ensure the parser can parse strings into Statement ASTs.
|
||||
func TestAstParser_ParseSet(t *testing.T) {
|
||||
var tests = []struct {
|
||||
in string
|
||||
tree ASTNode
|
||||
err error
|
||||
sql string
|
||||
args []interface{}
|
||||
}{
|
||||
{
|
||||
in: `log( arg1 ), arg2 / 100`,
|
||||
tree: ASTSet{
|
||||
Function{
|
||||
Name: "log",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "arg1"},
|
||||
},
|
||||
},
|
||||
|
||||
ASTNodes{
|
||||
Ident{Value: "arg2"},
|
||||
Operator{Kind: "/"},
|
||||
Number{Value: "100"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
in: `log( arg1 * 5 ), arg2 / 100 + 10`,
|
||||
tree: ASTSet{
|
||||
Function{
|
||||
Name: "log",
|
||||
Arguments: ASTSet{
|
||||
ASTNodes{
|
||||
Ident{Value: "arg1"},
|
||||
Operator{Kind: "*"},
|
||||
Number{Value: "5"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
ASTNodes{
|
||||
Ident{Value: "arg2"},
|
||||
Operator{Kind: "/"},
|
||||
Number{Value: "100"},
|
||||
Operator{Kind: "+"},
|
||||
Number{Value: "10"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
in: `date_format(created_at, '%Y')`,
|
||||
tree: ASTSet{
|
||||
Function{
|
||||
Name: "date_format",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "created_at"},
|
||||
String{Value: "%Y"},
|
||||
},
|
||||
},
|
||||
},
|
||||
sql: `date_format(created_at, ?)`,
|
||||
args: []interface{}{"%Y"},
|
||||
},
|
||||
// {
|
||||
// in: `year(created_at) > year(NOW()) - 2`,
|
||||
// tree: ASTSet{
|
||||
// Function{
|
||||
// Name: "year",
|
||||
// Arguments: ASTSet{
|
||||
// Ident{Value: "created_at"},
|
||||
// },
|
||||
// },
|
||||
// Operator{Kind: ">"},
|
||||
// Function{
|
||||
// Name: "year",
|
||||
// Arguments: ASTSet{
|
||||
// Function{
|
||||
// Name: "NOW",
|
||||
// },
|
||||
// },
|
||||
// },
|
||||
// Operator{Kind: "-"},
|
||||
// Number{Value: "2"},
|
||||
// },
|
||||
// sql: `year(created_at) > year(NOW()) - 2 `,
|
||||
// },
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
if tree, err := NewParser().ParseSet(test.in); err != test.err {
|
||||
t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err)
|
||||
} else if test.err == nil && !reflect.DeepEqual(test.tree, tree) {
|
||||
t.Errorf("%d. %s\n\ntree does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.tree, tree)
|
||||
} else if sql, args, err := tree.ToSql(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if test.sql != "" && sql != test.sql {
|
||||
t.Errorf("%d. %s\n\nsql does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.sql, sql)
|
||||
} else if test.args != nil && !reflect.DeepEqual(test.args, args) {
|
||||
t.Errorf("%d. %s\n\nargs does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.args, args)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the parser can parse strings into Statement ASTs.
|
||||
func TestAstParser_ParseExpression(t *testing.T) {
|
||||
var tests = []struct {
|
||||
in string
|
||||
tree ASTNode
|
||||
err error
|
||||
}{
|
||||
{
|
||||
in: `func(arg1, arg2)`,
|
||||
tree: Function{
|
||||
Name: "func",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "arg1"},
|
||||
Ident{Value: "arg2"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
in: `year(created_at) != 2010`,
|
||||
tree: ASTNodes{
|
||||
Function{
|
||||
Name: "year",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "created_at"},
|
||||
},
|
||||
},
|
||||
Operator{Kind: "!="},
|
||||
Number{Value: "2010"},
|
||||
},
|
||||
},
|
||||
{
|
||||
in: `year(created_at) != 2010 AND month(created_at) = 6`,
|
||||
tree: ASTNodes{
|
||||
Function{
|
||||
Name: "year",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "created_at"},
|
||||
},
|
||||
},
|
||||
Operator{Kind: "!="},
|
||||
Number{Value: "2010"},
|
||||
Keyword{"AND"},
|
||||
Function{
|
||||
Name: "month",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "created_at"},
|
||||
},
|
||||
},
|
||||
Operator{Kind: "="},
|
||||
Number{Value: "6"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
if tree, err := NewParser().ParseExpression(test.in); err != test.err {
|
||||
t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err)
|
||||
} else if test.err == nil && !reflect.DeepEqual(test.tree, tree) {
|
||||
t.Errorf("%d. %s\n\ntree does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.tree, tree)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAstParser_ColumnParser(t *testing.T) {
|
||||
var tests = []struct {
|
||||
in string
|
||||
cols Columns
|
||||
err error
|
||||
}{
|
||||
{
|
||||
in: `a AS b`,
|
||||
cols: Columns{
|
||||
Column{
|
||||
Expr: ASTNodes{Ident{Value: "a"}},
|
||||
Alias: "b",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
in: `sum(value1) as sumValue1, min(value2)`,
|
||||
cols: Columns{
|
||||
Column{
|
||||
Expr: ASTNodes{Function{
|
||||
Name: "sum",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "value1"},
|
||||
},
|
||||
}},
|
||||
Alias: "sumValue1",
|
||||
},
|
||||
Column{
|
||||
Expr: ASTNodes{Function{
|
||||
Name: "min",
|
||||
Arguments: ASTSet{
|
||||
Ident{Value: "value2"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
if cols, err := NewParser().ParseColumns(test.in); err != test.err {
|
||||
t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err)
|
||||
} else if test.err == nil && !reflect.DeepEqual(test.cols, cols) {
|
||||
t.Errorf("%d. %s\n\ncols does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.cols, cols)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAstParser_IdentModifier(t *testing.T) {
|
||||
var tests = []struct {
|
||||
in string
|
||||
out string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
in: `foo`,
|
||||
out: `__wrap_foo_wrap__`,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
p := NewParser()
|
||||
|
||||
p.OnIdent = func(ident Ident) (Ident, error) {
|
||||
ident.Value = fmt.Sprintf("__wrap_%s_wrap__", ident.Value)
|
||||
return ident, nil
|
||||
}
|
||||
|
||||
if tree, err := p.ParseExpression(test.in); err != test.err {
|
||||
t.Fatalf("%d. error mismatch:\n expected: %v\n got: %v\n\n", i, test.err, err)
|
||||
} else if test.err == nil && test.out != tree.String() {
|
||||
t.Errorf("%d. tree does not match:\n\n expected: %#v\n got: %#v\n\n", i, test.in, test.out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package ql
|
||||
|
||||
// MakeIdentWrapHandler makes an Ident wrapper
|
||||
//
|
||||
// Will return all skipped (ss) fields as-is and replace the rest with wrap, moving
|
||||
// ident (Value) to args
|
||||
func MakeIdentWrapHandler(wrap string, ss ...string) IdentHandler {
|
||||
return func(i Ident) (Ident, error) {
|
||||
for _, s := range ss {
|
||||
if s == i.Value {
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
|
||||
i.Args = []interface{}{i.Value}
|
||||
i.Value = wrap
|
||||
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package ql
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Lexer represents a lexical scanner.
|
||||
type (
|
||||
RuneReader interface {
|
||||
read() rune
|
||||
unread()
|
||||
}
|
||||
|
||||
TokenConsumers interface {
|
||||
Test(ch rune) bool
|
||||
Consume(s RuneReader) Token
|
||||
}
|
||||
|
||||
Token struct {
|
||||
code tokenCode
|
||||
literal string
|
||||
line uint
|
||||
char uint
|
||||
}
|
||||
|
||||
Lexer struct {
|
||||
r *bufio.Reader
|
||||
consumers []TokenConsumers
|
||||
line uint
|
||||
char uint
|
||||
}
|
||||
)
|
||||
|
||||
// eof represents a marker rune for the end of the reader.
|
||||
var eof = rune(0)
|
||||
|
||||
// NewLexer returns a new instance of Lexer.
|
||||
func NewLexer(r io.Reader) *Lexer {
|
||||
return &Lexer{
|
||||
r: bufio.NewReader(r),
|
||||
consumers: []TokenConsumers{
|
||||
&TokenConsumerGeneric{token: WS, whitelist: CHAR_WHITELIST_WHITESPACE},
|
||||
// @todo ensure operator order (eg != is valid, =! is not)
|
||||
&TokenConsumerGeneric{token: OPERATOR, whitelist: CHAR_WHITELIST_OPERATORS},
|
||||
&TokenConsumerGeneric{token: COMMA, whitelist: ","},
|
||||
&TokenConsumerGeneric{token: PARENTHESIS_OPEN, whitelist: "("},
|
||||
&TokenConsumerGeneric{token: PARENTHESIS_CLOSE, whitelist: ")"},
|
||||
&TokenConsumerString{},
|
||||
&TokenConsumerNumber{},
|
||||
&TokenConsumerIdent{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Scan returns the next token and literal value.
|
||||
func (s *Lexer) Scan() Token {
|
||||
var ch = s.peek()
|
||||
|
||||
if ch == '\n' {
|
||||
s.line++
|
||||
s.char = 0
|
||||
}
|
||||
|
||||
if eof == ch {
|
||||
return Token{code: EOF, line: s.line, char: s.char}
|
||||
}
|
||||
|
||||
for _, c := range s.consumers {
|
||||
if c.Test(ch) {
|
||||
t := c.Consume(s)
|
||||
t.line = s.line
|
||||
t.char = s.char
|
||||
return t
|
||||
}
|
||||
}
|
||||
|
||||
return Token{code: ILLEGAL, literal: string(ch), line: s.line, char: s.char}
|
||||
}
|
||||
|
||||
// read reads the next rune from the buffered reader.
|
||||
// Returns the rune(0) if an error occurs (or io.EOF is returned).
|
||||
func (s *Lexer) read() rune {
|
||||
ch, _, err := s.r.ReadRune()
|
||||
s.char++
|
||||
if err != nil {
|
||||
return eof
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
func (s *Lexer) peek() rune {
|
||||
bb, err := s.r.Peek(1)
|
||||
if err != nil || len(bb) == 0 {
|
||||
return eof
|
||||
}
|
||||
return rune(bb[0])
|
||||
}
|
||||
|
||||
// unread places the previously read rune back on the reader.
|
||||
func (s *Lexer) unread() { _ = s.r.UnreadRune() }
|
||||
|
||||
func (t Token) Is(cc ...tokenCode) bool {
|
||||
for _, c := range cc {
|
||||
if t.code == c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package ql
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Ensure the scanner can scan tokens correctly.
|
||||
func TestScanner_ScanSimple(t *testing.T) {
|
||||
var tests = []struct {
|
||||
s string
|
||||
tok tokenCode
|
||||
lit string
|
||||
}{
|
||||
// Special tokens (EOF, ILLEGAL, WS)
|
||||
{s: ``, tok: EOF},
|
||||
{s: `#`, tok: ILLEGAL, lit: `#`},
|
||||
{s: ` `, tok: WS, lit: " "},
|
||||
{s: "\t", tok: WS, lit: "\t"},
|
||||
{s: "\n", tok: WS, lit: "\n"},
|
||||
|
||||
// Operators
|
||||
{s: `*`, tok: OPERATOR, lit: "*"},
|
||||
{s: `!=`, tok: OPERATOR, lit: "!="},
|
||||
{s: `<`, tok: OPERATOR, lit: "<"},
|
||||
{s: `>`, tok: OPERATOR, lit: ">"},
|
||||
{s: `>=`, tok: OPERATOR, lit: ">="},
|
||||
{s: `<>`, tok: OPERATOR, lit: "<>"},
|
||||
{s: `+`, tok: OPERATOR, lit: "+"},
|
||||
{s: `'fooo'`, tok: STRING, lit: "fooo"},
|
||||
{s: `'escaped \' quote'`, tok: STRING, lit: "escaped ' quote"},
|
||||
{s: `'double \\ escape'`, tok: STRING, lit: "double \\ escape"},
|
||||
{s: `12345`, tok: NUMBER, lit: "12345"},
|
||||
|
||||
// Identifiers
|
||||
{s: `foo`, tok: IDENT, lit: `foo`},
|
||||
{s: `Zx12_3U_-`, tok: IDENT, lit: `Zx12_3U_`},
|
||||
|
||||
// Parenthesis
|
||||
{s: `(`, tok: PARENTHESIS_OPEN, lit: `(`},
|
||||
{s: `)`, tok: PARENTHESIS_CLOSE, lit: `)`},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
s := NewLexer(strings.NewReader(test.s))
|
||||
tok := s.Scan()
|
||||
if test.tok != tok.code {
|
||||
t.Errorf("%d. %q token mismatch: exp=%d got=%d <%q>", i, test.s, test.tok, tok.code, tok.literal)
|
||||
} else if test.lit != tok.literal {
|
||||
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, test.s, test.lit, tok.literal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanner_ScanComplex(t *testing.T) {
|
||||
var tests = []struct {
|
||||
s string
|
||||
tokens []tokenCode
|
||||
}{
|
||||
// Special tokens (EOF, ILLEGAL, WS)
|
||||
{`func(arg1, arg2)`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}},
|
||||
{`arg1 * arg2`, []tokenCode{IDENT, WS, OPERATOR, WS, IDENT}},
|
||||
{`date_format(created_at,'%Y')`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
var tokens []Token
|
||||
s := NewLexer(strings.NewReader(test.s))
|
||||
for {
|
||||
tok := s.Scan()
|
||||
if tok.Is(EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
tokens = append(tokens, tok)
|
||||
}
|
||||
|
||||
if len(tokens) != len(test.tokens) {
|
||||
t.Errorf("Collected tokens do not match (%v)", tokens)
|
||||
}
|
||||
|
||||
for i := 0; i < len(tokens); i++ {
|
||||
if tokens[i].code != test.tokens[i] {
|
||||
t.Errorf("Input: %s", test.s)
|
||||
t.Errorf("Expected: %v", test.tokens)
|
||||
t.Errorf("Collected: %v", tokens)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package ql
|
||||
|
||||
// Squirrel Sqlizer interface implementators for all ast node types
|
||||
// This helps us to throw columns into squirrel's select builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gopkg.in/Masterminds/squirrel.v1"
|
||||
)
|
||||
|
||||
// ToSql concatenates outputs and arguments from all nodes
|
||||
func (nn ASTNodes) ToSql() (out string, args []interface{}, err error) {
|
||||
var _out string
|
||||
var _args []interface{}
|
||||
|
||||
for _, s := range nn {
|
||||
if _out, _args, err = s.ToSql(); err != nil {
|
||||
return
|
||||
} else {
|
||||
out = out + _out
|
||||
|
||||
args = append(args, _args...)
|
||||
}
|
||||
}
|
||||
|
||||
return out, args, err
|
||||
}
|
||||
|
||||
// ToSql concatenates outputs and arguments from all nodes, comma delimited
|
||||
func (nn ASTSet) ToSql() (out string, args []interface{}, err error) {
|
||||
var _out string
|
||||
var _args []interface{}
|
||||
|
||||
for i, s := range nn {
|
||||
if _out, _args, err = s.ToSql(); err != nil {
|
||||
return
|
||||
} else {
|
||||
if i > 0 {
|
||||
out = out + ", "
|
||||
}
|
||||
|
||||
out = out + _out
|
||||
|
||||
args = append(args, _args...)
|
||||
}
|
||||
}
|
||||
|
||||
return out, args, err
|
||||
}
|
||||
|
||||
// ToSql returns column alias expression or outpit of underlaying expression's ToSql()
|
||||
func (n Column) ToSql() (string, []interface{}, error) {
|
||||
if n.Alias != "" {
|
||||
return squirrel.Alias(n.Expr, n.Alias).ToSql()
|
||||
} else {
|
||||
return n.Expr.ToSql()
|
||||
}
|
||||
}
|
||||
|
||||
func (n Ident) ToSql() (string, []interface{}, error) {
|
||||
return n.Value, n.Args, nil
|
||||
}
|
||||
|
||||
func (n Function) ToSql() (string, []interface{}, error) {
|
||||
if paramsSql, args, err := n.Arguments.ToSql(); err != nil {
|
||||
return "", nil, err
|
||||
} else {
|
||||
return fmt.Sprintf("%s(%s)", n.Name, paramsSql), args, nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (n Keyword) ToSql() (string, []interface{}, error) {
|
||||
return n.Keyword, nil, nil
|
||||
}
|
||||
|
||||
func (n Operator) ToSql() (string, []interface{}, error) {
|
||||
return n.Kind, nil, nil
|
||||
}
|
||||
|
||||
func (n String) ToSql() (string, []interface{}, error) {
|
||||
return "?", []interface{}{n.Value}, nil
|
||||
}
|
||||
|
||||
func (n Number) ToSql() (string, []interface{}, error) {
|
||||
return n.Value, nil, nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package ql
|
||||
|
||||
type (
|
||||
tokenCode int
|
||||
)
|
||||
|
||||
const (
|
||||
CHAR_WHITELIST_WHITESPACE = " \n\t"
|
||||
CHAR_WHITELIST_OPERATORS = "!+-/*=<>"
|
||||
CHAR_WHITELIST_QUOTES = "'"
|
||||
)
|
||||
|
||||
const (
|
||||
// Special tokens
|
||||
ILLEGAL tokenCode = iota
|
||||
EOF
|
||||
WS // 2
|
||||
IDENT
|
||||
NUMBER // 4
|
||||
STRING
|
||||
COMMA // ,
|
||||
OPERATOR // + - / *
|
||||
PARENTHESIS_OPEN
|
||||
PARENTHESIS_CLOSE
|
||||
)
|
||||
@@ -0,0 +1,139 @@
|
||||
package ql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
)
|
||||
|
||||
type (
|
||||
TokenConsumerWS struct{}
|
||||
TokenConsumerIdent struct{}
|
||||
TokenConsumerOperator struct{}
|
||||
TokenConsumerComma struct{}
|
||||
TokenConsumerString struct{}
|
||||
TokenConsumerNumber struct{}
|
||||
TokenConsumerGeneric struct {
|
||||
token tokenCode
|
||||
whitelist string
|
||||
}
|
||||
)
|
||||
|
||||
func in(ch rune, wl string) bool {
|
||||
var w rune
|
||||
for _, w = range wl {
|
||||
if ch == w {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (g TokenConsumerGeneric) Test(ch rune) bool {
|
||||
return in(ch, g.whitelist)
|
||||
}
|
||||
|
||||
func (g TokenConsumerGeneric) Consume(s RuneReader) Token {
|
||||
// Create a buffer and read the current character into it.
|
||||
var buf bytes.Buffer
|
||||
buf.WriteRune(s.read())
|
||||
|
||||
// Read every subsequent whitespace character into the buffer.
|
||||
// Non-whitespace characters and EOF will cause the loop to exit.
|
||||
for {
|
||||
if ch := s.read(); ch == eof {
|
||||
break
|
||||
} else if !g.Test(ch) {
|
||||
s.unread()
|
||||
break
|
||||
} else {
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return Token{code: g.token, literal: buf.String()}
|
||||
}
|
||||
|
||||
func (i TokenConsumerIdent) Test(ch rune) bool {
|
||||
return isLetter(ch)
|
||||
}
|
||||
|
||||
// Consumes the current rune and all contiguous ident runes.
|
||||
func (TokenConsumerIdent) Consume(s RuneReader) Token {
|
||||
// Create a buffer and read the current character into it.
|
||||
var buf bytes.Buffer
|
||||
buf.WriteRune(s.read())
|
||||
|
||||
// Read every subsequent ident character into the buffer.
|
||||
// Non-ident characters and EOF will cause the loop to exit.
|
||||
for {
|
||||
if ch := s.read(); ch == eof {
|
||||
break
|
||||
} else if !isLetter(ch) && !isDigit(ch) && ch != '_' {
|
||||
s.unread()
|
||||
break
|
||||
} else {
|
||||
_, _ = buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise return as a regular identifier.
|
||||
return Token{code: IDENT, literal: buf.String()}
|
||||
}
|
||||
|
||||
func (str TokenConsumerString) Test(ch rune) bool {
|
||||
return in(ch, CHAR_WHITELIST_QUOTES)
|
||||
}
|
||||
|
||||
// Consumes entire string (skipping quotes)
|
||||
func (str TokenConsumerString) Consume(s RuneReader) Token {
|
||||
var buf bytes.Buffer
|
||||
var escaping = false
|
||||
var ch = s.read() // skip quite
|
||||
|
||||
for {
|
||||
if ch = s.read(); ch == eof {
|
||||
break
|
||||
} else if !escaping && str.Test(ch) { // test for quote
|
||||
return Token{code: STRING, literal: buf.String()}
|
||||
} else {
|
||||
escaping = !escaping && ch == '\\'
|
||||
if !escaping {
|
||||
// Add char to buffer if not escaping
|
||||
_, _ = buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This string did not end properly (with an enclosing quote).
|
||||
return Token{code: ILLEGAL, literal: buf.String() + string(ch)}
|
||||
}
|
||||
|
||||
func (str TokenConsumerNumber) Test(ch rune) bool {
|
||||
return isDigit(ch)
|
||||
}
|
||||
|
||||
// Consumes entire number (very naive and simplified)
|
||||
func (str TokenConsumerNumber) Consume(s RuneReader) Token {
|
||||
// Create a buffer and read the current character into it.
|
||||
var buf bytes.Buffer
|
||||
buf.WriteRune(s.read())
|
||||
|
||||
for {
|
||||
if ch := s.read(); ch == eof {
|
||||
break
|
||||
} else if !isDigit(ch) {
|
||||
s.unread()
|
||||
break
|
||||
} else {
|
||||
_, _ = buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise return as a regular identifier.
|
||||
return Token{code: NUMBER, literal: buf.String()}
|
||||
}
|
||||
|
||||
// isLetter returns true if the rune is a letter.
|
||||
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
|
||||
|
||||
// isDigit returns true if the rune is a digit.
|
||||
func isDigit(ch rune) bool { return ch >= '0' && ch <= '9' }
|
||||
@@ -19,7 +19,7 @@ type (
|
||||
|
||||
FindByID(id uint64) (*types.Record, error)
|
||||
|
||||
Report(moduleID uint64, params *types.RecordReport) (results interface{}, err error)
|
||||
Report(moduleID uint64, metrics, dimensions, filter string) (results interface{}, err error)
|
||||
Find(moduleID uint64, query string, page int, perPage int, sort string) (*FindResponse, error)
|
||||
|
||||
Create(mod *types.Record) (*types.Record, error)
|
||||
@@ -47,6 +47,10 @@ type (
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
jsonWrap = `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`
|
||||
)
|
||||
|
||||
func Record(ctx context.Context, db *factory.DB) RecordRepository {
|
||||
return (&record{}).With(ctx, db)
|
||||
}
|
||||
@@ -67,8 +71,20 @@ func (r *record) FindByID(id uint64) (*types.Record, error) {
|
||||
return mod, nil
|
||||
}
|
||||
|
||||
func (r *record) Report(moduleID uint64, params *types.RecordReport) (results interface{}, err error) {
|
||||
crb := NewRecordReportBuilder(moduleID, params)
|
||||
func (r *record) Report(moduleID uint64, metrics, dimensions, filter string) (results interface{}, err error) {
|
||||
crb := NewRecordReportBuilder(moduleID)
|
||||
|
||||
if err = crb.SetMetrics(metrics); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = crb.SetDimensions(dimensions); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = crb.SetFilter(filter); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var result = make([]map[string]interface{}, 0)
|
||||
|
||||
|
||||
@@ -1,95 +1,92 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"gopkg.in/Masterminds/squirrel.v1"
|
||||
|
||||
"github.com/crusttech/crust/crm/types"
|
||||
"github.com/crusttech/crust/crm/repository/ql"
|
||||
)
|
||||
|
||||
type (
|
||||
recordReportBuilder struct {
|
||||
jsonField string
|
||||
|
||||
moduleID uint64
|
||||
params *types.RecordReport
|
||||
|
||||
metrics ql.Columns
|
||||
dimensions ql.Columns
|
||||
filter ql.ASTNode
|
||||
|
||||
// This is set by metric/column building to assist Cast()
|
||||
numerics []string
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
recordReportExprMatch = regexp.MustCompile(`^\s*(\w+)\((.+)\)\s*$`)
|
||||
)
|
||||
|
||||
func NewRecordReportBuilder(moduleID uint64, params *types.RecordReport) *recordReportBuilder {
|
||||
return &recordReportBuilder{
|
||||
moduleID: moduleID,
|
||||
params: params,
|
||||
jsonField: `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`,
|
||||
}
|
||||
}
|
||||
|
||||
func (b recordReportBuilder) field(name string) squirrel.Sqlizer {
|
||||
switch name {
|
||||
case "created_at", "updated_at":
|
||||
return squirrel.Expr(name)
|
||||
// Identifiers should be names of the fields (physical table columns OR json fields, defined in module)
|
||||
func stdAggregationHandler(f ql.Function) (ql.Function, error) {
|
||||
switch strings.ToUpper(f.Name) {
|
||||
// case "COUNTD":
|
||||
// return SqlConcatExpr("COUNT(DISTINCT ", aggrFuncArgs, ")")
|
||||
//
|
||||
case "COUNT", "SUM", "MAX", "MIN", "AVG", "STD":
|
||||
return f, nil
|
||||
default:
|
||||
return squirrel.Expr(b.jsonField, name)
|
||||
return f, fmt.Errorf("unsupported aggregate function %q", f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func (b recordReportBuilder) alias(col squirrel.Sqlizer, alias, fallback string) (squirrel.Sqlizer, string) {
|
||||
if alias != "" {
|
||||
return squirrel.Alias(col, alias), alias
|
||||
}
|
||||
|
||||
return squirrel.Alias(col, fallback), fallback
|
||||
}
|
||||
|
||||
func (b recordReportBuilder) wrapInModifiers(col squirrel.Sqlizer, mm ...string) squirrel.Sqlizer {
|
||||
for _, m := range mm {
|
||||
switch strings.ToUpper(m) {
|
||||
case "WEEKDAY":
|
||||
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%W')")
|
||||
case "DATE":
|
||||
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m-%d')")
|
||||
case "WEEK":
|
||||
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%u')")
|
||||
case "MONTH":
|
||||
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m')")
|
||||
case "QUARTER":
|
||||
col = SqlConcatExpr("CONCAT(", "YEAR(", col, "), 'Q', ", "QUARTER(", col, ")", ")")
|
||||
case "YEAR":
|
||||
col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y')")
|
||||
// Identifiers should be names of the fields (physical table columns OR json fields, defined in module)
|
||||
func stdGroupByFuncHandler(f ql.Function) (ql.Function, error) {
|
||||
switch strings.ToUpper(f.Name) {
|
||||
case "DATE_FORMAT":
|
||||
if len(f.Arguments) == 2 {
|
||||
return f, nil
|
||||
} else {
|
||||
return f, fmt.Errorf("incorrect parameter count for group-by function '%s'", f.Name)
|
||||
}
|
||||
case "CONCAT", "QUARTER", "YEAR", "DATE", "NOW":
|
||||
return f, nil
|
||||
|
||||
default:
|
||||
return f, fmt.Errorf("unsupported group-by function %q", f.Name)
|
||||
}
|
||||
|
||||
return col
|
||||
}
|
||||
|
||||
func (b recordReportBuilder) parseExpression(exp string) squirrel.Sqlizer {
|
||||
res := recordReportExprMatch.FindStringSubmatch(exp)
|
||||
if len(res) > 0 {
|
||||
aggrFuncName := strings.ToUpper(res[1])
|
||||
aggrFuncArgs := b.parseExpression(res[2])
|
||||
func NewRecordReportBuilder(moduleID uint64) *recordReportBuilder {
|
||||
return &recordReportBuilder{moduleID: moduleID}
|
||||
}
|
||||
|
||||
switch aggrFuncName {
|
||||
case "COUNTD":
|
||||
return SqlConcatExpr("COUNT(DISTINCT ", aggrFuncArgs, ")")
|
||||
func (b *recordReportBuilder) SetMetrics(metrics string) (err error) {
|
||||
p := ql.NewParser()
|
||||
|
||||
case "SUM", "MAX", "MIN", "AVG", "STD":
|
||||
return SqlConcatExpr(aggrFuncName+"(CAST(", aggrFuncArgs, " AS DECIMAL(14,2)))")
|
||||
}
|
||||
} else {
|
||||
return b.field(exp)
|
||||
}
|
||||
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at")
|
||||
p.OnFunction = stdAggregationHandler
|
||||
|
||||
return nil
|
||||
b.metrics, err = p.ParseColumns(metrics)
|
||||
return
|
||||
}
|
||||
|
||||
func (b *recordReportBuilder) SetDimensions(dimensions string) (err error) {
|
||||
p := ql.NewParser()
|
||||
|
||||
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at")
|
||||
p.OnFunction = stdGroupByFuncHandler
|
||||
|
||||
b.dimensions, err = p.ParseColumns(dimensions)
|
||||
return
|
||||
}
|
||||
|
||||
func (b *recordReportBuilder) SetFilter(filters string) (err error) {
|
||||
p := ql.NewParser()
|
||||
|
||||
p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at", "id", "user_id")
|
||||
p.OnFunction = stdGroupByFuncHandler
|
||||
|
||||
b.filter, err = p.ParseExpression(filters)
|
||||
return
|
||||
}
|
||||
|
||||
func (b *recordReportBuilder) Build() (sql string, args []interface{}, err error) {
|
||||
@@ -99,33 +96,33 @@ func (b *recordReportBuilder) Build() (sql string, args []interface{}, err error
|
||||
From("crm_record").
|
||||
Where("module_id = ?", b.moduleID)
|
||||
|
||||
if b.params == nil {
|
||||
return "", nil, errors.New("can not generate report without parameters")
|
||||
}
|
||||
|
||||
for i, m := range b.params.Metrics {
|
||||
col := SqlConcatExpr("CAST(", b.parseExpression(m.Expression), " AS DECIMAL(14,2))")
|
||||
col, m.Alias = b.alias(col, m.Alias, fmt.Sprintf("metric_%d", i))
|
||||
// Add all metrics to columns
|
||||
for i, m := range b.metrics {
|
||||
if m.Alias == "" {
|
||||
// Generate alias
|
||||
m.Alias = fmt.Sprintf("metric_%d", i)
|
||||
}
|
||||
|
||||
// Wrap to cast func to ensure numeric output
|
||||
col := squirrel.Alias(SqlConcatExpr("CAST(", m.Expr, " AS DECIMAL(14,2))"), m.Alias)
|
||||
report = report.Column(col)
|
||||
|
||||
b.params.Metrics[i].Alias = m.Alias // copy generated alias back
|
||||
b.numerics = append(b.numerics, m.Alias)
|
||||
}
|
||||
|
||||
for i, d := range b.params.Dimensions {
|
||||
col := b.field(d.Field)
|
||||
// Add all dimensions to columns
|
||||
for i, d := range b.dimensions {
|
||||
if d.Alias == "" {
|
||||
d.Alias = fmt.Sprintf("dimension_%d", i)
|
||||
}
|
||||
|
||||
col = b.wrapInModifiers(col, d.Modifiers...)
|
||||
|
||||
col, d.Alias = b.alias(col, d.Alias, fmt.Sprintf("dimension_%d", i))
|
||||
|
||||
report = report.Column(col)
|
||||
report = report.Column(d)
|
||||
report = report.GroupBy(d.Alias)
|
||||
report = report.OrderBy(d.Alias)
|
||||
|
||||
b.params.Dimensions[i].Alias = d.Alias // copy generated alias back
|
||||
}
|
||||
|
||||
report = report.Where(b.filter)
|
||||
|
||||
return report.ToSql()
|
||||
}
|
||||
|
||||
@@ -140,8 +137,8 @@ func (b recordReportBuilder) Cast(row sqlx.ColScanner) map[string]interface{} {
|
||||
}
|
||||
|
||||
// Cast all metrics to float64
|
||||
for _, m := range b.params.Metrics {
|
||||
out[m.Alias], _ = strconv.ParseFloat(out[m.Alias].(string), 64)
|
||||
for _, numeric := range b.numerics {
|
||||
out[numeric], _ = strconv.ParseFloat(out[numeric].(string), 64)
|
||||
}
|
||||
|
||||
return out
|
||||
|
||||
+1
-12
@@ -2,7 +2,6 @@ package rest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/titpetric/factory/resputil"
|
||||
|
||||
@@ -55,17 +54,7 @@ func (s *Module) Edit(ctx context.Context, r *request.ModuleEdit) (interface{},
|
||||
}
|
||||
|
||||
func (s *Module) RecordReport(ctx context.Context, r *request.ModuleRecordReport) (interface{}, error) {
|
||||
reportParams := &types.RecordReport{}
|
||||
|
||||
if strings.TrimSpace(r.Metrics) != "" {
|
||||
reportParams.ScanMetrics(strings.Split(r.Metrics, ",")...)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(r.Dimensions) != "" {
|
||||
reportParams.ScanDimensions(strings.Split(r.Dimensions, ",")...)
|
||||
}
|
||||
|
||||
return s.content.With(ctx).Report(r.ModuleID, reportParams)
|
||||
return s.content.With(ctx).Report(r.ModuleID, r.Metrics, r.Dimensions, r.Filter)
|
||||
}
|
||||
|
||||
func (s *Module) RecordList(ctx context.Context, r *request.ModuleRecordList) (interface{}, error) {
|
||||
|
||||
@@ -264,6 +264,7 @@ var _ RequestFiller = NewModuleDelete()
|
||||
type ModuleRecordReport struct {
|
||||
Metrics string
|
||||
Dimensions string
|
||||
Filter string
|
||||
ModuleID uint64 `json:",string"`
|
||||
}
|
||||
|
||||
@@ -306,6 +307,10 @@ func (m *ModuleRecordReport) Fill(r *http.Request) (err error) {
|
||||
|
||||
m.Dimensions = val
|
||||
}
|
||||
if val, ok := get["filter"]; ok {
|
||||
|
||||
m.Filter = val
|
||||
}
|
||||
m.ModuleID = parseUInt64(chi.URLParam(r, "moduleID"))
|
||||
|
||||
return err
|
||||
|
||||
@@ -28,7 +28,7 @@ type (
|
||||
|
||||
FindByID(recordID uint64) (*types.Record, error)
|
||||
|
||||
Report(moduleID uint64, params *types.RecordReport) (interface{}, error)
|
||||
Report(moduleID uint64, metrics, dimensions, filter string) (interface{}, error)
|
||||
Find(moduleID uint64, query string, page int, perPage int, sort string) (*repository.FindResponse, error)
|
||||
|
||||
Create(record *types.Record) (*types.Record, error)
|
||||
@@ -64,8 +64,8 @@ func (s *record) FindByID(id uint64) (*types.Record, error) {
|
||||
return response, s.preload(response, "page", "user", "fields")
|
||||
}
|
||||
|
||||
func (s *record) Report(moduleID uint64, params *types.RecordReport) (interface{}, error) {
|
||||
return s.repository.Report(moduleID, params)
|
||||
func (s *record) Report(moduleID uint64, metrics, dimensions, filter string) (interface{}, error) {
|
||||
return s.repository.Report(moduleID, metrics, dimensions, filter)
|
||||
}
|
||||
|
||||
func (s *record) Find(moduleID uint64, query string, page int, perPage int, sort string) (*repository.FindResponse, error) {
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type (
|
||||
RecordReport struct {
|
||||
Metrics []RecordReportMetric
|
||||
Dimensions []RecordReportDimensions
|
||||
}
|
||||
|
||||
RecordReportMetric struct {
|
||||
Alias string
|
||||
Expression string
|
||||
}
|
||||
|
||||
RecordReportDimensions struct {
|
||||
Alias string
|
||||
Field string
|
||||
Modifiers []string
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
recordReportMetricScanRE *regexp.Regexp
|
||||
recordReportDimensionScanRE *regexp.Regexp
|
||||
)
|
||||
|
||||
func init() {
|
||||
recordReportMetricScanRE = regexp.MustCompile("^(?:(\\w+):)?(.+)$")
|
||||
recordReportDimensionScanRE = regexp.MustCompile("^(?:(\\w+):)?(\\w+)((?:\\|?\\w+)+)?$")
|
||||
}
|
||||
|
||||
func (r *RecordReport) ScanMetrics(metrics ...string) {
|
||||
r.Metrics = make([]RecordReportMetric, len(metrics))
|
||||
for i := 0; i < len(metrics); i++ {
|
||||
r.Metrics[i].Scan(metrics[i])
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RecordReport) ScanDimensions(dimensions ...string) {
|
||||
r.Dimensions = make([]RecordReportDimensions, len(dimensions))
|
||||
for i := 0; i < len(dimensions); i++ {
|
||||
r.Dimensions[i].Scan(dimensions[i])
|
||||
}
|
||||
}
|
||||
|
||||
func (m *RecordReportMetric) Scan(metric string) {
|
||||
if match := recordReportMetricScanRE.FindStringSubmatch(metric); len(match) == 3 {
|
||||
m.Alias = match[1]
|
||||
m.Expression = match[2]
|
||||
}
|
||||
}
|
||||
|
||||
func (d *RecordReportDimensions) Scan(dimension string) {
|
||||
if match := recordReportDimensionScanRE.FindStringSubmatch(dimension); len(match) == 4 {
|
||||
d.Alias = match[1]
|
||||
d.Field = match[2]
|
||||
|
||||
if len(match[3]) > 0 {
|
||||
d.Modifiers = strings.Split(match[3][1:], "|")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRecordReport(t *testing.T) {
|
||||
r := &RecordReport{}
|
||||
r.ScanMetrics("alias:exp")
|
||||
if len(r.Metrics) == 0 {
|
||||
t.Log("No metrics scanned")
|
||||
t.Fail()
|
||||
} else if r.Metrics[0].Alias != "alias" {
|
||||
t.Log("Alias not parsed")
|
||||
t.Fail()
|
||||
} else if r.Metrics[0].Expression != "exp" {
|
||||
t.Log("Expression not parsed")
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
r.ScanMetrics("exp")
|
||||
if len(r.Metrics) == 0 {
|
||||
t.Log("No metrics scanned")
|
||||
t.Fail()
|
||||
} else if r.Metrics[0].Alias != "" {
|
||||
t.Log("Alias should be empty")
|
||||
t.Fail()
|
||||
} else if r.Metrics[0].Expression != "exp" {
|
||||
t.Log("Expression not parsed")
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
r.ScanDimensions("alias:field|m1|m2")
|
||||
if len(r.Dimensions) == 0 {
|
||||
t.Log("No dimensions scanned")
|
||||
t.Fail()
|
||||
} else if r.Dimensions[0].Alias != "alias" {
|
||||
t.Log("Alias not parsed")
|
||||
t.Fail()
|
||||
} else if r.Dimensions[0].Field != "field" {
|
||||
t.Log("Expression not parsed")
|
||||
t.Fail()
|
||||
} else if len(r.Dimensions[0].Modifiers) != 2 && r.Dimensions[0].Modifiers[0] != "m1" || r.Dimensions[0].Modifiers[1] != "m2" {
|
||||
t.Log("Modifiers not parsed")
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
r.ScanDimensions("field")
|
||||
if len(r.Dimensions) == 0 {
|
||||
t.Log("No dimensions scanned")
|
||||
t.Fail()
|
||||
} else if r.Dimensions[0].Alias != "" {
|
||||
t.Log("Alias should be empty")
|
||||
t.Fail()
|
||||
} else if len(r.Dimensions[0].Modifiers) > 0 {
|
||||
t.Log("Modifiers should be empty")
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
+3
-2
@@ -300,8 +300,9 @@ CRM module definitions
|
||||
|
||||
| Parameter | Type | Method | Description | Default | Required? |
|
||||
| --------- | ---- | ------ | ----------- | ------- | --------- |
|
||||
| metrics | string | GET | Metrics (syntax: alias:expression;...) | N/A | YES |
|
||||
| dimensions | string | GET | Dimensions (syntax: alias:field|modifier|modifier2;...) | N/A | YES |
|
||||
| metrics | string | GET | Metrics (eg: 'COUNT(*) AS count, SUM(money)') | N/A | YES |
|
||||
| dimensions | string | GET | Dimensions (eg: 'DATE(foo), status') | N/A | YES |
|
||||
| filter | string | GET | Filter (eg: 'DATE(foo) > 2010') | N/A | YES |
|
||||
| moduleID | uint64 | PATH | Module ID | N/A | YES |
|
||||
|
||||
## List/read records from module section
|
||||
|
||||
Reference in New Issue
Block a user