diff --git a/api/crm/spec.json b/api/crm/spec.json index f9ed61f41..6f6f6571d 100644 --- a/api/crm/spec.json +++ b/api/crm/spec.json @@ -676,13 +676,19 @@ "type": "string", "name": "metrics", "required": true, - "title": "Metrics (syntax: alias:expression;...)" + "title": "Metrics (eg: 'COUNT(*) AS count, SUM(money)')" }, { "type": "string", "name": "dimensions", "required": true, - "title": "Dimensions (syntax: alias:field|modifier|modifier2;...)" + "title": "Dimensions (eg: 'DATE(foo), status')" + }, + { + "type": "string", + "name": "filter", + "required": true, + "title": "Filter (eg: 'DATE(foo) > 2010')" } ] } diff --git a/api/crm/spec/module.json b/api/crm/spec/module.json index 4376a9417..139a43e76 100644 --- a/api/crm/spec/module.json +++ b/api/crm/spec/module.json @@ -126,13 +126,19 @@ { "name": "metrics", "required": true, - "title": "Metrics (syntax: alias:expression;...)", + "title": "Metrics (eg: 'COUNT(*) AS count, SUM(money)')", "type": "string" }, { "name": "dimensions", "required": true, - "title": "Dimensions (syntax: alias:field|modifier|modifier2;...)", + "title": "Dimensions (eg: 'DATE(foo), status')", + "type": "string" + }, + { + "name": "filter", + "required": true, + "title": "Filter (eg: 'DATE(foo) \u003e 2010')", "type": "string" } ], diff --git a/crm/repository/ql/README.md b/crm/repository/ql/README.md new file mode 100644 index 000000000..5794cc2cd --- /dev/null +++ b/crm/repository/ql/README.md @@ -0,0 +1,28 @@ +# QueryLanguage (ql) package + +_This package was written mainly to assist with column +conversion (wrapping non-physical columns into functions that +extract values from a JSON value)_ + +Provides lexer and ast parser to convert a set of simple +instructions (see `ast_parser_test.go` for examples) that closely +resemble SQL into syntax trees. + +It provides handlers for identifiers and functions that can +validate and modify nodes and tree at parse time. + +Tree can than be converted back to SQL or to structs that +assist Squirrel select builder. + +## Pending improvements + +### Lexer + + - Operator validation + +### AST Parser + + - simplify / combine ASTNode vs ASTSet vs Columns + - improve resilience and detect basic syntax errors + - properly handling COUNT(DISTINCT ...) ident syntax + - parsing complex expressions (eg: `year(created_at) > year(NOW()) - 2`) diff --git a/crm/repository/ql/ast_nodes.go b/crm/repository/ql/ast_nodes.go new file mode 100644 index 000000000..62e8b4eb9 --- /dev/null +++ b/crm/repository/ql/ast_nodes.go @@ -0,0 +1,102 @@ +package ql + +import ( + "fmt" + + "gopkg.in/Masterminds/squirrel.v1" +) + +// SelectStatement represents a SQL SELECT statement. +type ( + ASTNode interface { + fmt.Stringer + squirrel.Sqlizer + } + + ASTSet []ASTNode // Stream of comma delimited nodes + ASTNodes []ASTNode // Stream of un-delimited nodes + + Columns []Column + + String struct { + Value string + Args []interface{} + } + + Number struct { + Value string + } + + Operator struct { + Kind string + } + + Ident struct { + Value string + Args []interface{} + } + + Keyword struct { + Keyword string + } + + Column struct { + Expr ASTNodes + Alias string + } + Function struct { + Name string + Arguments ASTSet + } +) + +func (n String) String() string { return fmt.Sprintf("%q", n.Value) } + +func (n Number) String() string { return n.Value } + +func (n Operator) String() string { return n.Kind } + +func (n Keyword) String() string { return n.Keyword } + +func (n Function) String() string { return fmt.Sprintf("%s(%s)", n.Name, n.Arguments) } + +func (n Ident) String() string { return n.Value } + +func (n Column) String() (out string) { + out = n.Expr.String() + if n.Alias != "" { + out = out + " AS " + n.Alias + } + + return +} + +func (nn ASTNodes) String() (out string) { + for _, n := range nn { + out = out + n.String() + } + + return +} + +func (nn ASTSet) String() (out string) { + for i, n := range nn { + if i > 0 { + out = out + ", " + } + out = out + n.String() + } + + return +} + +func (nn Columns) String() (out string) { + for i, n := range nn { + if i > 0 { + out = out + ", " + } + out = out + n.String() + } + + return +} diff --git a/crm/repository/ql/ast_parser.go b/crm/repository/ql/ast_parser.go new file mode 100644 index 000000000..30cce4ec5 --- /dev/null +++ b/crm/repository/ql/ast_parser.go @@ -0,0 +1,276 @@ +package ql + +import ( + "fmt" + "strings" +) + +type ( + // Parser represents a parser. + Parser struct { + lexer *Lexer + tokbuf []Token + + OnIdent IdentHandler + OnFunction FunctionHandler + + // parenthesis level control + level uint + } + + IdentHandler func(ident Ident) (Ident, error) + FunctionHandler func(ident Function) (Function, error) +) + +// NewParser returns a new instance of Parser. +func NewParser() *Parser { + p := &Parser{ + tokbuf: make([]Token, 3), + OnIdent: func(ident Ident) (Ident, error) { return ident, nil }, + OnFunction: func(ident Function) (Function, error) { return ident, nil }, + } + + return p +} + +// Removes oldest token in the buffer, adds new one and returns 2nd oldest +func (p *Parser) nextToken() Token { + var t Token + for { + t = p.lexer.Scan() + if !t.Is(WS) { + p.tokbuf = append(p.tokbuf[1:], t) + return p.tokbuf[0] + } + } +} + +func (p *Parser) peekToken(s int) Token { + return p.tokbuf[s] +} + +func (p *Parser) initLexer(s string) { + p.lexer = NewLexer(strings.NewReader(s)) + + for c := 1; c < cap(p.tokbuf); c++ { + // Fill the buffer + p.nextToken() + } +} + +func (p *Parser) ParseSet(s string) (ASTNode, error) { + p.initLexer(s) + + if set, err := p.parseSet(); err != nil { + return nil, err + } else { + return set, nil + } +} + +func (p *Parser) ParseExpression(s string) (ASTNode, error) { + p.initLexer(s) + + if set, err := p.parseExpr(p.nextToken()); err != nil { + return nil, err + } else if len(set) == 1 { + return set[0], nil + } else { + return set, nil + } +} + +func (p *Parser) ParseColumns(s string) (columns Columns, err error) { + p.initLexer(s) + + var t Token + var c Column + +next: + t = p.nextToken() + switch t.code { + case COMMA: + goto next + case EOF: + break + case ILLEGAL: + return nil, fmt.Errorf("found an illegal token (%+v)", t) + default: + if c, err = p.parseColumn(t); err != nil { + return nil, err + } else { + columns = append(columns, c) + } + goto next + } + + return +} + +func (p *Parser) parseColumn(t Token) (c Column, err error) { + if c.Expr, err = p.parseExpr(t); err != nil { + return + } + + // Set alias move forward for 2 places + if p.peekIfAlias() { + c.Alias = p.peekToken(2).literal + p.nextToken() + p.nextToken() + } + + return +} + +// Peek ahead if there is an alias ident ( +func (p *Parser) peekIfAlias() bool { + var f, s = p.peekToken(1), p.peekToken(2) + return f.Is(IDENT) && strings.ToUpper(f.literal) == "AS" && s.Is(IDENT) +} + +func (p *Parser) parseExpr(t Token) (list ASTNodes, err error) { + goto checkToken + +next: + if p.peekToken(1).Is(COMMA) || p.peekIfAlias() { + // Peek ahead and exit on comma + return + } + if p.peekToken(1).Is(PARENTHESIS_CLOSE) { + // Peek ahead and exit on closed parenthesis + if p.level == 0 { + return nil, fmt.Errorf("closing unopened parenthesis in expression") + } + p.level-- + return + } + + t = p.nextToken() + +checkToken: + switch t.code { + case EOF: + break + case WS: + // Ignore ws... next token... + goto next + case ILLEGAL: + return nil, fmt.Errorf("found an illegal token (%+v)", t) + case IDENT: + var ident ASTNode + if ident, err = p.parseIdent(t); err != nil { + return nil, err + } else { + list = append(list, ident) + + goto next + } + case OPERATOR: + list = append(list, Operator{Kind: t.literal}) + goto next + case NUMBER: + list = append(list, Number{Value: t.literal}) + goto next + case STRING: + list = append(list, String{Value: t.literal}) + goto next + case PARENTHESIS_OPEN: + p.level++ + if sub, err := p.parseExpr(p.nextToken()); err != nil { + return nil, err + } else { + list = append(list, sub) + } + default: + return nil, fmt.Errorf("unexpected token while parsing expression (%v)", t) + } + + return list, nil +} + +func (p *Parser) parseIdent(t Token) (list ASTNode, err error) { + var ucLiteral = strings.ToUpper(t.literal) + + if ucLiteral == "AND" || ucLiteral == "OR" { + return Keyword{ucLiteral}, nil + } else if p.peekToken(1).Is(PARENTHESIS_OPEN) { + // Handle function calls: ... + f := Function{Name: t.literal} + if f.Arguments, err = p.parseSet(); err != nil { + return nil, err + } else { + return p.OnFunction(f) + } + } + + return p.OnIdent(Ident{Value: t.literal}) +} + +func (p *Parser) parseSet() (list ASTSet, err error) { + var expr ASTNodes + var parenthesisOpened = false + +next: + t := p.nextToken() + + if p.peekIfAlias() { + return + } + + switch t.code { + case WS: + goto next + case PARENTHESIS_OPEN: + p.level++ + parenthesisOpened = true + goto next + case NUMBER: + list = append(list, Number{Value: t.literal}) + goto next + case STRING: + list = append(list, String{Value: t.literal}) + goto next + case OPERATOR: + list = append(list, Operator{Kind: t.literal}) + goto next + case IDENT: + if p.peekToken(1).Is(OPERATOR) { + // Looks like we have an expression ahead of us + + if parenthesisOpened { + // Expression will find closing parenthesis and dec. the level + // so, lets bump up the number + p.level++ + } + + if expr, err = p.parseExpr(t); err != nil { + return + } + + list = append(list, expr) + goto next + } + + var ident ASTNode + if ident, err = p.parseIdent(t); err != nil { + return nil, err + } else { + list = append(list, ident) + + goto next + } + case COMMA: + goto next + case EOF: + return + case PARENTHESIS_CLOSE: + // Peek ahead and exit on closed parenthesis + if p.level == 0 { + return nil, fmt.Errorf("closing unopened parenthesis in set") + } + p.level-- + return + default: + return nil, fmt.Errorf("unexpected token while parsing set (%v)", t) + } +} diff --git a/crm/repository/ql/ast_parser_test.go b/crm/repository/ql/ast_parser_test.go new file mode 100644 index 000000000..8da0505e8 --- /dev/null +++ b/crm/repository/ql/ast_parser_test.go @@ -0,0 +1,250 @@ +package ql + +import ( + "fmt" + "reflect" + "testing" +) + +// Ensure the parser can parse strings into Statement ASTs. +func TestAstParser_ParseSet(t *testing.T) { + var tests = []struct { + in string + tree ASTNode + err error + sql string + args []interface{} + }{ + { + in: `log( arg1 ), arg2 / 100`, + tree: ASTSet{ + Function{ + Name: "log", + Arguments: ASTSet{ + Ident{Value: "arg1"}, + }, + }, + + ASTNodes{ + Ident{Value: "arg2"}, + Operator{Kind: "/"}, + Number{Value: "100"}, + }, + }, + }, + { + in: `log( arg1 * 5 ), arg2 / 100 + 10`, + tree: ASTSet{ + Function{ + Name: "log", + Arguments: ASTSet{ + ASTNodes{ + Ident{Value: "arg1"}, + Operator{Kind: "*"}, + Number{Value: "5"}, + }, + }, + }, + + ASTNodes{ + Ident{Value: "arg2"}, + Operator{Kind: "/"}, + Number{Value: "100"}, + Operator{Kind: "+"}, + Number{Value: "10"}, + }, + }, + }, + { + in: `date_format(created_at, '%Y')`, + tree: ASTSet{ + Function{ + Name: "date_format", + Arguments: ASTSet{ + Ident{Value: "created_at"}, + String{Value: "%Y"}, + }, + }, + }, + sql: `date_format(created_at, ?)`, + args: []interface{}{"%Y"}, + }, + // { + // in: `year(created_at) > year(NOW()) - 2`, + // tree: ASTSet{ + // Function{ + // Name: "year", + // Arguments: ASTSet{ + // Ident{Value: "created_at"}, + // }, + // }, + // Operator{Kind: ">"}, + // Function{ + // Name: "year", + // Arguments: ASTSet{ + // Function{ + // Name: "NOW", + // }, + // }, + // }, + // Operator{Kind: "-"}, + // Number{Value: "2"}, + // }, + // sql: `year(created_at) > year(NOW()) - 2 `, + // }, + } + + for i, test := range tests { + if tree, err := NewParser().ParseSet(test.in); err != test.err { + t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err) + } else if test.err == nil && !reflect.DeepEqual(test.tree, tree) { + t.Errorf("%d. %s\n\ntree does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.tree, tree) + } else if sql, args, err := tree.ToSql(); err != nil { + t.Fatal(err) + } else if test.sql != "" && sql != test.sql { + t.Errorf("%d. %s\n\nsql does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.sql, sql) + } else if test.args != nil && !reflect.DeepEqual(test.args, args) { + t.Errorf("%d. %s\n\nargs does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.args, args) + + } + } +} + +// Ensure the parser can parse strings into Statement ASTs. +func TestAstParser_ParseExpression(t *testing.T) { + var tests = []struct { + in string + tree ASTNode + err error + }{ + { + in: `func(arg1, arg2)`, + tree: Function{ + Name: "func", + Arguments: ASTSet{ + Ident{Value: "arg1"}, + Ident{Value: "arg2"}, + }, + }, + }, + { + in: `year(created_at) != 2010`, + tree: ASTNodes{ + Function{ + Name: "year", + Arguments: ASTSet{ + Ident{Value: "created_at"}, + }, + }, + Operator{Kind: "!="}, + Number{Value: "2010"}, + }, + }, + { + in: `year(created_at) != 2010 AND month(created_at) = 6`, + tree: ASTNodes{ + Function{ + Name: "year", + Arguments: ASTSet{ + Ident{Value: "created_at"}, + }, + }, + Operator{Kind: "!="}, + Number{Value: "2010"}, + Keyword{"AND"}, + Function{ + Name: "month", + Arguments: ASTSet{ + Ident{Value: "created_at"}, + }, + }, + Operator{Kind: "="}, + Number{Value: "6"}, + }, + }, + } + + for i, test := range tests { + if tree, err := NewParser().ParseExpression(test.in); err != test.err { + t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err) + } else if test.err == nil && !reflect.DeepEqual(test.tree, tree) { + t.Errorf("%d. %s\n\ntree does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.tree, tree) + } + } +} + +func TestAstParser_ColumnParser(t *testing.T) { + var tests = []struct { + in string + cols Columns + err error + }{ + { + in: `a AS b`, + cols: Columns{ + Column{ + Expr: ASTNodes{Ident{Value: "a"}}, + Alias: "b", + }, + }, + }, + { + in: `sum(value1) as sumValue1, min(value2)`, + cols: Columns{ + Column{ + Expr: ASTNodes{Function{ + Name: "sum", + Arguments: ASTSet{ + Ident{Value: "value1"}, + }, + }}, + Alias: "sumValue1", + }, + Column{ + Expr: ASTNodes{Function{ + Name: "min", + Arguments: ASTSet{ + Ident{Value: "value2"}, + }, + }}, + }, + }, + }, + } + + for i, test := range tests { + if cols, err := NewParser().ParseColumns(test.in); err != test.err { + t.Fatalf("%d. %s: error mismatch:\n expected: %v\n got: %v\n\n", i, test.in, test.err, err) + } else if test.err == nil && !reflect.DeepEqual(test.cols, cols) { + t.Errorf("%d. %s\n\ncols does not match:\n\nexpected: %#v\n got: %#v\n\n", i, test.in, test.cols, cols) + } + } +} + +func TestAstParser_IdentModifier(t *testing.T) { + var tests = []struct { + in string + out string + err error + }{ + { + in: `foo`, + out: `__wrap_foo_wrap__`, + }, + } + + for i, test := range tests { + p := NewParser() + + p.OnIdent = func(ident Ident) (Ident, error) { + ident.Value = fmt.Sprintf("__wrap_%s_wrap__", ident.Value) + return ident, nil + } + + if tree, err := p.ParseExpression(test.in); err != test.err { + t.Fatalf("%d. error mismatch:\n expected: %v\n got: %v\n\n", i, test.err, err) + } else if test.err == nil && test.out != tree.String() { + t.Errorf("%d. tree does not match:\n\n expected: %#v\n got: %#v\n\n", i, test.in, test.out) + } + } +} diff --git a/crm/repository/ql/handlers.go b/crm/repository/ql/handlers.go new file mode 100644 index 000000000..ed205db2b --- /dev/null +++ b/crm/repository/ql/handlers.go @@ -0,0 +1,20 @@ +package ql + +// MakeIdentWrapHandler makes an Ident wrapper +// +// Will return all skipped (ss) fields as-is and replace the rest with wrap, moving +// ident (Value) to args +func MakeIdentWrapHandler(wrap string, ss ...string) IdentHandler { + return func(i Ident) (Ident, error) { + for _, s := range ss { + if s == i.Value { + return i, nil + } + } + + i.Args = []interface{}{i.Value} + i.Value = wrap + + return i, nil + } +} diff --git a/crm/repository/ql/lexer.go b/crm/repository/ql/lexer.go new file mode 100644 index 000000000..a102bb694 --- /dev/null +++ b/crm/repository/ql/lexer.go @@ -0,0 +1,111 @@ +package ql + +import ( + "bufio" + "io" +) + +// Lexer represents a lexical scanner. +type ( + RuneReader interface { + read() rune + unread() + } + + TokenConsumers interface { + Test(ch rune) bool + Consume(s RuneReader) Token + } + + Token struct { + code tokenCode + literal string + line uint + char uint + } + + Lexer struct { + r *bufio.Reader + consumers []TokenConsumers + line uint + char uint + } +) + +// eof represents a marker rune for the end of the reader. +var eof = rune(0) + +// NewLexer returns a new instance of Lexer. +func NewLexer(r io.Reader) *Lexer { + return &Lexer{ + r: bufio.NewReader(r), + consumers: []TokenConsumers{ + &TokenConsumerGeneric{token: WS, whitelist: CHAR_WHITELIST_WHITESPACE}, + // @todo ensure operator order (eg != is valid, =! is not) + &TokenConsumerGeneric{token: OPERATOR, whitelist: CHAR_WHITELIST_OPERATORS}, + &TokenConsumerGeneric{token: COMMA, whitelist: ","}, + &TokenConsumerGeneric{token: PARENTHESIS_OPEN, whitelist: "("}, + &TokenConsumerGeneric{token: PARENTHESIS_CLOSE, whitelist: ")"}, + &TokenConsumerString{}, + &TokenConsumerNumber{}, + &TokenConsumerIdent{}, + }, + } +} + +// Scan returns the next token and literal value. +func (s *Lexer) Scan() Token { + var ch = s.peek() + + if ch == '\n' { + s.line++ + s.char = 0 + } + + if eof == ch { + return Token{code: EOF, line: s.line, char: s.char} + } + + for _, c := range s.consumers { + if c.Test(ch) { + t := c.Consume(s) + t.line = s.line + t.char = s.char + return t + } + } + + return Token{code: ILLEGAL, literal: string(ch), line: s.line, char: s.char} +} + +// read reads the next rune from the buffered reader. +// Returns the rune(0) if an error occurs (or io.EOF is returned). +func (s *Lexer) read() rune { + ch, _, err := s.r.ReadRune() + s.char++ + if err != nil { + return eof + } + return ch +} + +func (s *Lexer) peek() rune { + bb, err := s.r.Peek(1) + if err != nil || len(bb) == 0 { + return eof + } + return rune(bb[0]) +} + +// unread places the previously read rune back on the reader. +func (s *Lexer) unread() { _ = s.r.UnreadRune() } + +func (t Token) Is(cc ...tokenCode) bool { + for _, c := range cc { + if t.code == c { + return true + } + } + + return false +} diff --git a/crm/repository/ql/lexer_test.go b/crm/repository/ql/lexer_test.go new file mode 100644 index 000000000..e775cc61a --- /dev/null +++ b/crm/repository/ql/lexer_test.go @@ -0,0 +1,91 @@ +package ql + +import ( + "strings" + "testing" +) + +// Ensure the scanner can scan tokens correctly. +func TestScanner_ScanSimple(t *testing.T) { + var tests = []struct { + s string + tok tokenCode + lit string + }{ + // Special tokens (EOF, ILLEGAL, WS) + {s: ``, tok: EOF}, + {s: `#`, tok: ILLEGAL, lit: `#`}, + {s: ` `, tok: WS, lit: " "}, + {s: "\t", tok: WS, lit: "\t"}, + {s: "\n", tok: WS, lit: "\n"}, + + // Operators + {s: `*`, tok: OPERATOR, lit: "*"}, + {s: `!=`, tok: OPERATOR, lit: "!="}, + {s: `<`, tok: OPERATOR, lit: "<"}, + {s: `>`, tok: OPERATOR, lit: ">"}, + {s: `>=`, tok: OPERATOR, lit: ">="}, + {s: `<>`, tok: OPERATOR, lit: "<>"}, + {s: `+`, tok: OPERATOR, lit: "+"}, + {s: `'fooo'`, tok: STRING, lit: "fooo"}, + {s: `'escaped \' quote'`, tok: STRING, lit: "escaped ' quote"}, + {s: `'double \\ escape'`, tok: STRING, lit: "double \\ escape"}, + {s: `12345`, tok: NUMBER, lit: "12345"}, + + // Identifiers + {s: `foo`, tok: IDENT, lit: `foo`}, + {s: `Zx12_3U_-`, tok: IDENT, lit: `Zx12_3U_`}, + + // Parenthesis + {s: `(`, tok: PARENTHESIS_OPEN, lit: `(`}, + {s: `)`, tok: PARENTHESIS_CLOSE, lit: `)`}, + } + + for i, test := range tests { + s := NewLexer(strings.NewReader(test.s)) + tok := s.Scan() + if test.tok != tok.code { + t.Errorf("%d. %q token mismatch: exp=%d got=%d <%q>", i, test.s, test.tok, tok.code, tok.literal) + } else if test.lit != tok.literal { + t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, test.s, test.lit, tok.literal) + } + } +} + +func TestScanner_ScanComplex(t *testing.T) { + var tests = []struct { + s string + tokens []tokenCode + }{ + // Special tokens (EOF, ILLEGAL, WS) + {`func(arg1, arg2)`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}}, + {`arg1 * arg2`, []tokenCode{IDENT, WS, OPERATOR, WS, IDENT}}, + {`date_format(created_at,'%Y')`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}}, + } + + for _, test := range tests { + var tokens []Token + s := NewLexer(strings.NewReader(test.s)) + for { + tok := s.Scan() + if tok.Is(EOF) { + break + } + + tokens = append(tokens, tok) + } + + if len(tokens) != len(test.tokens) { + t.Errorf("Collected tokens do not match (%v)", tokens) + } + + for i := 0; i < len(tokens); i++ { + if tokens[i].code != test.tokens[i] { + t.Errorf("Input: %s", test.s) + t.Errorf("Expected: %v", test.tokens) + t.Errorf("Collected: %v", tokens) + break + } + } + } +} diff --git a/crm/repository/ql/squirrel.go b/crm/repository/ql/squirrel.go new file mode 100644 index 000000000..32073d158 --- /dev/null +++ b/crm/repository/ql/squirrel.go @@ -0,0 +1,88 @@ +package ql + +// Squirrel Sqlizer interface implementators for all ast node types +// This helps us to throw columns into squirrel's select builder + +import ( + "fmt" + + "gopkg.in/Masterminds/squirrel.v1" +) + +// ToSql concatenates outputs and arguments from all nodes +func (nn ASTNodes) ToSql() (out string, args []interface{}, err error) { + var _out string + var _args []interface{} + + for _, s := range nn { + if _out, _args, err = s.ToSql(); err != nil { + return + } else { + out = out + _out + + args = append(args, _args...) + } + } + + return out, args, err +} + +// ToSql concatenates outputs and arguments from all nodes, comma delimited +func (nn ASTSet) ToSql() (out string, args []interface{}, err error) { + var _out string + var _args []interface{} + + for i, s := range nn { + if _out, _args, err = s.ToSql(); err != nil { + return + } else { + if i > 0 { + out = out + ", " + } + + out = out + _out + + args = append(args, _args...) + } + } + + return out, args, err +} + +// ToSql returns column alias expression or outpit of underlaying expression's ToSql() +func (n Column) ToSql() (string, []interface{}, error) { + if n.Alias != "" { + return squirrel.Alias(n.Expr, n.Alias).ToSql() + } else { + return n.Expr.ToSql() + } +} + +func (n Ident) ToSql() (string, []interface{}, error) { + return n.Value, n.Args, nil +} + +func (n Function) ToSql() (string, []interface{}, error) { + if paramsSql, args, err := n.Arguments.ToSql(); err != nil { + return "", nil, err + } else { + return fmt.Sprintf("%s(%s)", n.Name, paramsSql), args, nil + } + +} + +func (n Keyword) ToSql() (string, []interface{}, error) { + return n.Keyword, nil, nil +} + +func (n Operator) ToSql() (string, []interface{}, error) { + return n.Kind, nil, nil +} + +func (n String) ToSql() (string, []interface{}, error) { + return "?", []interface{}{n.Value}, nil +} + +func (n Number) ToSql() (string, []interface{}, error) { + return n.Value, nil, nil +} diff --git a/crm/repository/ql/token_codes.go b/crm/repository/ql/token_codes.go new file mode 100644 index 000000000..fd7d7c6f4 --- /dev/null +++ b/crm/repository/ql/token_codes.go @@ -0,0 +1,25 @@ +package ql + +type ( + tokenCode int +) + +const ( + CHAR_WHITELIST_WHITESPACE = " \n\t" + CHAR_WHITELIST_OPERATORS = "!+-/*=<>" + CHAR_WHITELIST_QUOTES = "'" +) + +const ( + // Special tokens + ILLEGAL tokenCode = iota + EOF + WS // 2 + IDENT + NUMBER // 4 + STRING + COMMA // , + OPERATOR // + - / * + PARENTHESIS_OPEN + PARENTHESIS_CLOSE +) diff --git a/crm/repository/ql/token_consumers.go b/crm/repository/ql/token_consumers.go new file mode 100644 index 000000000..cdb66a05a --- /dev/null +++ b/crm/repository/ql/token_consumers.go @@ -0,0 +1,139 @@ +package ql + +import ( + "bytes" +) + +type ( + TokenConsumerWS struct{} + TokenConsumerIdent struct{} + TokenConsumerOperator struct{} + TokenConsumerComma struct{} + TokenConsumerString struct{} + TokenConsumerNumber struct{} + TokenConsumerGeneric struct { + token tokenCode + whitelist string + } +) + +func in(ch rune, wl string) bool { + var w rune + for _, w = range wl { + if ch == w { + return true + } + } + return false +} + +func (g TokenConsumerGeneric) Test(ch rune) bool { + return in(ch, g.whitelist) +} + +func (g TokenConsumerGeneric) Consume(s RuneReader) Token { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + // Read every subsequent whitespace character into the buffer. + // Non-whitespace characters and EOF will cause the loop to exit. + for { + if ch := s.read(); ch == eof { + break + } else if !g.Test(ch) { + s.unread() + break + } else { + buf.WriteRune(ch) + } + } + + return Token{code: g.token, literal: buf.String()} +} + +func (i TokenConsumerIdent) Test(ch rune) bool { + return isLetter(ch) +} + +// Consumes the current rune and all contiguous ident runes. +func (TokenConsumerIdent) Consume(s RuneReader) Token { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + // Read every subsequent ident character into the buffer. + // Non-ident characters and EOF will cause the loop to exit. + for { + if ch := s.read(); ch == eof { + break + } else if !isLetter(ch) && !isDigit(ch) && ch != '_' { + s.unread() + break + } else { + _, _ = buf.WriteRune(ch) + } + } + + // Otherwise return as a regular identifier. + return Token{code: IDENT, literal: buf.String()} +} + +func (str TokenConsumerString) Test(ch rune) bool { + return in(ch, CHAR_WHITELIST_QUOTES) +} + +// Consumes entire string (skipping quotes) +func (str TokenConsumerString) Consume(s RuneReader) Token { + var buf bytes.Buffer + var escaping = false + var ch = s.read() // skip quite + + for { + if ch = s.read(); ch == eof { + break + } else if !escaping && str.Test(ch) { // test for quote + return Token{code: STRING, literal: buf.String()} + } else { + escaping = !escaping && ch == '\\' + if !escaping { + // Add char to buffer if not escaping + _, _ = buf.WriteRune(ch) + } + } + } + + // This string did not end properly (with an enclosing quote). + return Token{code: ILLEGAL, literal: buf.String() + string(ch)} +} + +func (str TokenConsumerNumber) Test(ch rune) bool { + return isDigit(ch) +} + +// Consumes entire number (very naive and simplified) +func (str TokenConsumerNumber) Consume(s RuneReader) Token { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + for { + if ch := s.read(); ch == eof { + break + } else if !isDigit(ch) { + s.unread() + break + } else { + _, _ = buf.WriteRune(ch) + } + } + + // Otherwise return as a regular identifier. + return Token{code: NUMBER, literal: buf.String()} +} + +// isLetter returns true if the rune is a letter. +func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } + +// isDigit returns true if the rune is a digit. +func isDigit(ch rune) bool { return ch >= '0' && ch <= '9' } diff --git a/crm/repository/record.go b/crm/repository/record.go index 8cb8692d2..ea08087db 100644 --- a/crm/repository/record.go +++ b/crm/repository/record.go @@ -19,7 +19,7 @@ type ( FindByID(id uint64) (*types.Record, error) - Report(moduleID uint64, params *types.RecordReport) (results interface{}, err error) + Report(moduleID uint64, metrics, dimensions, filter string) (results interface{}, err error) Find(moduleID uint64, query string, page int, perPage int, sort string) (*FindResponse, error) Create(mod *types.Record) (*types.Record, error) @@ -47,6 +47,10 @@ type ( } ) +const ( + jsonWrap = `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))` +) + func Record(ctx context.Context, db *factory.DB) RecordRepository { return (&record{}).With(ctx, db) } @@ -67,8 +71,20 @@ func (r *record) FindByID(id uint64) (*types.Record, error) { return mod, nil } -func (r *record) Report(moduleID uint64, params *types.RecordReport) (results interface{}, err error) { - crb := NewRecordReportBuilder(moduleID, params) +func (r *record) Report(moduleID uint64, metrics, dimensions, filter string) (results interface{}, err error) { + crb := NewRecordReportBuilder(moduleID) + + if err = crb.SetMetrics(metrics); err != nil { + return + } + + if err = crb.SetDimensions(dimensions); err != nil { + return + } + + if err = crb.SetFilter(filter); err != nil { + return + } var result = make([]map[string]interface{}, 0) diff --git a/crm/repository/record_report_builder.go b/crm/repository/record_report_builder.go index 7ab878e68..10bb532f9 100644 --- a/crm/repository/record_report_builder.go +++ b/crm/repository/record_report_builder.go @@ -1,95 +1,92 @@ package repository import ( - "errors" "fmt" - "regexp" "strconv" "strings" "github.com/jmoiron/sqlx" "gopkg.in/Masterminds/squirrel.v1" - "github.com/crusttech/crust/crm/types" + "github.com/crusttech/crust/crm/repository/ql" ) type ( recordReportBuilder struct { - jsonField string - moduleID uint64 - params *types.RecordReport + + metrics ql.Columns + dimensions ql.Columns + filter ql.ASTNode + + // This is set by metric/column building to assist Cast() + numerics []string } ) -var ( - recordReportExprMatch = regexp.MustCompile(`^\s*(\w+)\((.+)\)\s*$`) -) - -func NewRecordReportBuilder(moduleID uint64, params *types.RecordReport) *recordReportBuilder { - return &recordReportBuilder{ - moduleID: moduleID, - params: params, - jsonField: `JSON_UNQUOTE(JSON_EXTRACT(json, REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', ?)), '.name', '.value')))`, - } -} - -func (b recordReportBuilder) field(name string) squirrel.Sqlizer { - switch name { - case "created_at", "updated_at": - return squirrel.Expr(name) +// Identifiers should be names of the fields (physical table columns OR json fields, defined in module) +func stdAggregationHandler(f ql.Function) (ql.Function, error) { + switch strings.ToUpper(f.Name) { + // case "COUNTD": + // return SqlConcatExpr("COUNT(DISTINCT ", aggrFuncArgs, ")") + // + case "COUNT", "SUM", "MAX", "MIN", "AVG", "STD": + return f, nil default: - return squirrel.Expr(b.jsonField, name) + return f, fmt.Errorf("unsupported aggregate function %q", f.Name) } } -func (b recordReportBuilder) alias(col squirrel.Sqlizer, alias, fallback string) (squirrel.Sqlizer, string) { - if alias != "" { - return squirrel.Alias(col, alias), alias - } - - return squirrel.Alias(col, fallback), fallback -} - -func (b recordReportBuilder) wrapInModifiers(col squirrel.Sqlizer, mm ...string) squirrel.Sqlizer { - for _, m := range mm { - switch strings.ToUpper(m) { - case "WEEKDAY": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%W')") - case "DATE": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m-%d')") - case "WEEK": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%u')") - case "MONTH": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y-%m')") - case "QUARTER": - col = SqlConcatExpr("CONCAT(", "YEAR(", col, "), 'Q', ", "QUARTER(", col, ")", ")") - case "YEAR": - col = SqlConcatExpr("DATE_FORMAT(", col, ", '%Y')") +// Identifiers should be names of the fields (physical table columns OR json fields, defined in module) +func stdGroupByFuncHandler(f ql.Function) (ql.Function, error) { + switch strings.ToUpper(f.Name) { + case "DATE_FORMAT": + if len(f.Arguments) == 2 { + return f, nil + } else { + return f, fmt.Errorf("incorrect parameter count for group-by function '%s'", f.Name) } + case "CONCAT", "QUARTER", "YEAR", "DATE", "NOW": + return f, nil + + default: + return f, fmt.Errorf("unsupported group-by function %q", f.Name) } - return col } -func (b recordReportBuilder) parseExpression(exp string) squirrel.Sqlizer { - res := recordReportExprMatch.FindStringSubmatch(exp) - if len(res) > 0 { - aggrFuncName := strings.ToUpper(res[1]) - aggrFuncArgs := b.parseExpression(res[2]) +func NewRecordReportBuilder(moduleID uint64) *recordReportBuilder { + return &recordReportBuilder{moduleID: moduleID} +} - switch aggrFuncName { - case "COUNTD": - return SqlConcatExpr("COUNT(DISTINCT ", aggrFuncArgs, ")") +func (b *recordReportBuilder) SetMetrics(metrics string) (err error) { + p := ql.NewParser() - case "SUM", "MAX", "MIN", "AVG", "STD": - return SqlConcatExpr(aggrFuncName+"(CAST(", aggrFuncArgs, " AS DECIMAL(14,2)))") - } - } else { - return b.field(exp) - } + p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at") + p.OnFunction = stdAggregationHandler - return nil + b.metrics, err = p.ParseColumns(metrics) + return +} + +func (b *recordReportBuilder) SetDimensions(dimensions string) (err error) { + p := ql.NewParser() + + p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at") + p.OnFunction = stdGroupByFuncHandler + + b.dimensions, err = p.ParseColumns(dimensions) + return +} + +func (b *recordReportBuilder) SetFilter(filters string) (err error) { + p := ql.NewParser() + + p.OnIdent = ql.MakeIdentWrapHandler(jsonWrap, "created_at", "updated_at", "id", "user_id") + p.OnFunction = stdGroupByFuncHandler + + b.filter, err = p.ParseExpression(filters) + return } func (b *recordReportBuilder) Build() (sql string, args []interface{}, err error) { @@ -99,33 +96,33 @@ func (b *recordReportBuilder) Build() (sql string, args []interface{}, err error From("crm_record"). Where("module_id = ?", b.moduleID) - if b.params == nil { - return "", nil, errors.New("can not generate report without parameters") - } - - for i, m := range b.params.Metrics { - col := SqlConcatExpr("CAST(", b.parseExpression(m.Expression), " AS DECIMAL(14,2))") - col, m.Alias = b.alias(col, m.Alias, fmt.Sprintf("metric_%d", i)) + // Add all metrics to columns + for i, m := range b.metrics { + if m.Alias == "" { + // Generate alias + m.Alias = fmt.Sprintf("metric_%d", i) + } + // Wrap to cast func to ensure numeric output + col := squirrel.Alias(SqlConcatExpr("CAST(", m.Expr, " AS DECIMAL(14,2))"), m.Alias) report = report.Column(col) - b.params.Metrics[i].Alias = m.Alias // copy generated alias back + b.numerics = append(b.numerics, m.Alias) } - for i, d := range b.params.Dimensions { - col := b.field(d.Field) + // Add all dimensions to columns + for i, d := range b.dimensions { + if d.Alias == "" { + d.Alias = fmt.Sprintf("dimension_%d", i) + } - col = b.wrapInModifiers(col, d.Modifiers...) - - col, d.Alias = b.alias(col, d.Alias, fmt.Sprintf("dimension_%d", i)) - - report = report.Column(col) + report = report.Column(d) report = report.GroupBy(d.Alias) report = report.OrderBy(d.Alias) - - b.params.Dimensions[i].Alias = d.Alias // copy generated alias back } + report = report.Where(b.filter) + return report.ToSql() } @@ -140,8 +137,8 @@ func (b recordReportBuilder) Cast(row sqlx.ColScanner) map[string]interface{} { } // Cast all metrics to float64 - for _, m := range b.params.Metrics { - out[m.Alias], _ = strconv.ParseFloat(out[m.Alias].(string), 64) + for _, numeric := range b.numerics { + out[numeric], _ = strconv.ParseFloat(out[numeric].(string), 64) } return out diff --git a/crm/rest/module.go b/crm/rest/module.go index 897788a9d..c5e5bf6cf 100644 --- a/crm/rest/module.go +++ b/crm/rest/module.go @@ -2,7 +2,6 @@ package rest import ( "context" - "strings" "github.com/titpetric/factory/resputil" @@ -55,17 +54,7 @@ func (s *Module) Edit(ctx context.Context, r *request.ModuleEdit) (interface{}, } func (s *Module) RecordReport(ctx context.Context, r *request.ModuleRecordReport) (interface{}, error) { - reportParams := &types.RecordReport{} - - if strings.TrimSpace(r.Metrics) != "" { - reportParams.ScanMetrics(strings.Split(r.Metrics, ",")...) - } - - if strings.TrimSpace(r.Dimensions) != "" { - reportParams.ScanDimensions(strings.Split(r.Dimensions, ",")...) - } - - return s.content.With(ctx).Report(r.ModuleID, reportParams) + return s.content.With(ctx).Report(r.ModuleID, r.Metrics, r.Dimensions, r.Filter) } func (s *Module) RecordList(ctx context.Context, r *request.ModuleRecordList) (interface{}, error) { diff --git a/crm/rest/request/module.go b/crm/rest/request/module.go index f61d8abbc..aef5844cc 100644 --- a/crm/rest/request/module.go +++ b/crm/rest/request/module.go @@ -264,6 +264,7 @@ var _ RequestFiller = NewModuleDelete() type ModuleRecordReport struct { Metrics string Dimensions string + Filter string ModuleID uint64 `json:",string"` } @@ -306,6 +307,10 @@ func (m *ModuleRecordReport) Fill(r *http.Request) (err error) { m.Dimensions = val } + if val, ok := get["filter"]; ok { + + m.Filter = val + } m.ModuleID = parseUInt64(chi.URLParam(r, "moduleID")) return err diff --git a/crm/service/record.go b/crm/service/record.go index 34afc668d..0a49463c9 100644 --- a/crm/service/record.go +++ b/crm/service/record.go @@ -28,7 +28,7 @@ type ( FindByID(recordID uint64) (*types.Record, error) - Report(moduleID uint64, params *types.RecordReport) (interface{}, error) + Report(moduleID uint64, metrics, dimensions, filter string) (interface{}, error) Find(moduleID uint64, query string, page int, perPage int, sort string) (*repository.FindResponse, error) Create(record *types.Record) (*types.Record, error) @@ -64,8 +64,8 @@ func (s *record) FindByID(id uint64) (*types.Record, error) { return response, s.preload(response, "page", "user", "fields") } -func (s *record) Report(moduleID uint64, params *types.RecordReport) (interface{}, error) { - return s.repository.Report(moduleID, params) +func (s *record) Report(moduleID uint64, metrics, dimensions, filter string) (interface{}, error) { + return s.repository.Report(moduleID, metrics, dimensions, filter) } func (s *record) Find(moduleID uint64, query string, page int, perPage int, sort string) (*repository.FindResponse, error) { diff --git a/crm/types/record_util.go b/crm/types/record_util.go deleted file mode 100644 index 4cb4ad794..000000000 --- a/crm/types/record_util.go +++ /dev/null @@ -1,66 +0,0 @@ -package types - -import ( - "regexp" - "strings" -) - -type ( - RecordReport struct { - Metrics []RecordReportMetric - Dimensions []RecordReportDimensions - } - - RecordReportMetric struct { - Alias string - Expression string - } - - RecordReportDimensions struct { - Alias string - Field string - Modifiers []string - } -) - -var ( - recordReportMetricScanRE *regexp.Regexp - recordReportDimensionScanRE *regexp.Regexp -) - -func init() { - recordReportMetricScanRE = regexp.MustCompile("^(?:(\\w+):)?(.+)$") - recordReportDimensionScanRE = regexp.MustCompile("^(?:(\\w+):)?(\\w+)((?:\\|?\\w+)+)?$") -} - -func (r *RecordReport) ScanMetrics(metrics ...string) { - r.Metrics = make([]RecordReportMetric, len(metrics)) - for i := 0; i < len(metrics); i++ { - r.Metrics[i].Scan(metrics[i]) - } -} - -func (r *RecordReport) ScanDimensions(dimensions ...string) { - r.Dimensions = make([]RecordReportDimensions, len(dimensions)) - for i := 0; i < len(dimensions); i++ { - r.Dimensions[i].Scan(dimensions[i]) - } -} - -func (m *RecordReportMetric) Scan(metric string) { - if match := recordReportMetricScanRE.FindStringSubmatch(metric); len(match) == 3 { - m.Alias = match[1] - m.Expression = match[2] - } -} - -func (d *RecordReportDimensions) Scan(dimension string) { - if match := recordReportDimensionScanRE.FindStringSubmatch(dimension); len(match) == 4 { - d.Alias = match[1] - d.Field = match[2] - - if len(match[3]) > 0 { - d.Modifiers = strings.Split(match[3][1:], "|") - } - } -} diff --git a/crm/types/record_util_test.go b/crm/types/record_util_test.go deleted file mode 100644 index 7dac53231..000000000 --- a/crm/types/record_util_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package types - -import ( - "testing" -) - -func TestRecordReport(t *testing.T) { - r := &RecordReport{} - r.ScanMetrics("alias:exp") - if len(r.Metrics) == 0 { - t.Log("No metrics scanned") - t.Fail() - } else if r.Metrics[0].Alias != "alias" { - t.Log("Alias not parsed") - t.Fail() - } else if r.Metrics[0].Expression != "exp" { - t.Log("Expression not parsed") - t.Fail() - } - - r.ScanMetrics("exp") - if len(r.Metrics) == 0 { - t.Log("No metrics scanned") - t.Fail() - } else if r.Metrics[0].Alias != "" { - t.Log("Alias should be empty") - t.Fail() - } else if r.Metrics[0].Expression != "exp" { - t.Log("Expression not parsed") - t.Fail() - } - - r.ScanDimensions("alias:field|m1|m2") - if len(r.Dimensions) == 0 { - t.Log("No dimensions scanned") - t.Fail() - } else if r.Dimensions[0].Alias != "alias" { - t.Log("Alias not parsed") - t.Fail() - } else if r.Dimensions[0].Field != "field" { - t.Log("Expression not parsed") - t.Fail() - } else if len(r.Dimensions[0].Modifiers) != 2 && r.Dimensions[0].Modifiers[0] != "m1" || r.Dimensions[0].Modifiers[1] != "m2" { - t.Log("Modifiers not parsed") - t.Fail() - } - - r.ScanDimensions("field") - if len(r.Dimensions) == 0 { - t.Log("No dimensions scanned") - t.Fail() - } else if r.Dimensions[0].Alias != "" { - t.Log("Alias should be empty") - t.Fail() - } else if len(r.Dimensions[0].Modifiers) > 0 { - t.Log("Modifiers should be empty") - t.Fail() - } -} diff --git a/docs/crm/README.md b/docs/crm/README.md index 39181da19..33b11bf72 100644 --- a/docs/crm/README.md +++ b/docs/crm/README.md @@ -300,8 +300,9 @@ CRM module definitions | Parameter | Type | Method | Description | Default | Required? | | --------- | ---- | ------ | ----------- | ------- | --------- | -| metrics | string | GET | Metrics (syntax: alias:expression;...) | N/A | YES | -| dimensions | string | GET | Dimensions (syntax: alias:field|modifier|modifier2;...) | N/A | YES | +| metrics | string | GET | Metrics (eg: 'COUNT(*) AS count, SUM(money)') | N/A | YES | +| dimensions | string | GET | Dimensions (eg: 'DATE(foo), status') | N/A | YES | +| filter | string | GET | Filter (eg: 'DATE(foo) > 2010') | N/A | YES | | moduleID | uint64 | PATH | Module ID | N/A | YES | ## List/read records from module section