diff --git a/crm/repository/ql/ast_parser.go b/crm/repository/ql/ast_parser.go index 30cce4ec5..065926d25 100644 --- a/crm/repository/ql/ast_parser.go +++ b/crm/repository/ql/ast_parser.go @@ -162,12 +162,22 @@ checkToken: return nil, err } else { list = append(list, ident) - goto next } case OPERATOR: + if len(list) > 0 { + // Merge with previous operator node + if prevOp, ok := list[len(list)-1].(Operator); ok { + list[len(list)-1] = Operator{Kind: prevOp.Kind + " " + t.literal} + goto next + } + } + list = append(list, Operator{Kind: t.literal}) goto next + case KEYWORD: + list = append(list, Keyword{Keyword: t.literal}) + goto next case NUMBER: list = append(list, Number{Value: t.literal}) goto next @@ -189,11 +199,7 @@ checkToken: } func (p *Parser) parseIdent(t Token) (list ASTNode, err error) { - var ucLiteral = strings.ToUpper(t.literal) - - if ucLiteral == "AND" || ucLiteral == "OR" { - return Keyword{ucLiteral}, nil - } else if p.peekToken(1).Is(PARENTHESIS_OPEN) { + if p.peekToken(1).Is(PARENTHESIS_OPEN) { // Handle function calls: ... f := Function{Name: t.literal} if f.Arguments, err = p.parseSet(); err != nil { diff --git a/crm/repository/ql/ast_parser_test.go b/crm/repository/ql/ast_parser_test.go index 8da0505e8..04e916398 100644 --- a/crm/repository/ql/ast_parser_test.go +++ b/crm/repository/ql/ast_parser_test.go @@ -151,7 +151,7 @@ func TestAstParser_ParseExpression(t *testing.T) { }, Operator{Kind: "!="}, Number{Value: "2010"}, - Keyword{"AND"}, + Operator{"AND"}, Function{ Name: "month", Arguments: ASTSet{ @@ -162,6 +162,22 @@ func TestAstParser_ParseExpression(t *testing.T) { Number{Value: "6"}, }, }, + { + in: `foo LIKE 'bar%'`, + tree: ASTNodes{ + Ident{Value: "foo"}, + Operator{Kind: "LIKE"}, + String{Value: "bar%"}, + }, + }, + { + in: `foo NOT LIKE 'bar%'`, + tree: ASTNodes{ + Ident{Value: "foo"}, + Operator{Kind: "NOT LIKE"}, + String{Value: "bar%"}, + }, + }, } for i, test := range tests { @@ -210,6 +226,22 @@ func TestAstParser_ColumnParser(t *testing.T) { }, }, }, + { + in: `a DESC`, + cols: Columns{ + Column{ + Expr: ASTNodes{Ident{Value: "a"}, Keyword{Keyword: "DESC"}}, + }, + }, + }, + { + in: `a ASC`, + cols: Columns{ + Column{ + Expr: ASTNodes{Ident{Value: "a"}, Keyword{Keyword: "ASC"}}, + }, + }, + }, } for i, test := range tests { diff --git a/crm/repository/ql/lexer_test.go b/crm/repository/ql/lexer_test.go index e775cc61a..123509290 100644 --- a/crm/repository/ql/lexer_test.go +++ b/crm/repository/ql/lexer_test.go @@ -58,9 +58,18 @@ func TestScanner_ScanComplex(t *testing.T) { tokens []tokenCode }{ // Special tokens (EOF, ILLEGAL, WS) - {`func(arg1, arg2)`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}}, - {`arg1 * arg2`, []tokenCode{IDENT, WS, OPERATOR, WS, IDENT}}, - {`date_format(created_at,'%Y')`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}}, + {`func(arg1, arg2)`, + []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}}, + {`arg1 * arg2`, + []tokenCode{IDENT, WS, OPERATOR, WS, IDENT}}, + {`date_format(created_at,'%Y')`, + []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}}, + {`foo LIKE 'abc%'`, + []tokenCode{IDENT, WS, OPERATOR, WS, STRING}}, + {`foo NOT LIKE 'abc%'`, + []tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, STRING}}, + {`foo DESC`, + []tokenCode{IDENT, WS, KEYWORD}}, } for _, test := range tests { diff --git a/crm/repository/ql/squirrel.go b/crm/repository/ql/squirrel.go index 32073d158..446cca346 100644 --- a/crm/repository/ql/squirrel.go +++ b/crm/repository/ql/squirrel.go @@ -76,7 +76,15 @@ func (n Keyword) ToSql() (string, []interface{}, error) { } func (n Operator) ToSql() (string, []interface{}, error) { - return n.Kind, nil, nil + var op = n.Kind + + switch n.Kind { + case "LIKE", "NOT LIKE": + // Make sure we are doing case insensitive search + op = "COLLATE utf8mb4_general_ci " + n.Kind + } + + return " " + op + " ", nil, nil } func (n String) ToSql() (string, []interface{}, error) { diff --git a/crm/repository/ql/token_codes.go b/crm/repository/ql/token_codes.go index fd7d7c6f4..b1d996cc5 100644 --- a/crm/repository/ql/token_codes.go +++ b/crm/repository/ql/token_codes.go @@ -22,4 +22,5 @@ const ( OPERATOR // + - / * PARENTHESIS_OPEN PARENTHESIS_CLOSE + KEYWORD ) diff --git a/crm/repository/ql/token_consumers.go b/crm/repository/ql/token_consumers.go index cdb66a05a..dc0d9a550 100644 --- a/crm/repository/ql/token_consumers.go +++ b/crm/repository/ql/token_consumers.go @@ -2,6 +2,7 @@ package ql import ( "bytes" + "strings" ) type ( @@ -75,6 +76,15 @@ func (TokenConsumerIdent) Consume(s RuneReader) Token { } } + lit := strings.ToUpper(buf.String()) + + switch lit { + case "LIKE", "NOT", "AND", "OR", "XOR": + return Token{code: OPERATOR, literal: lit} + case "DESC", "ASC": + return Token{code: KEYWORD, literal: lit} + } + // Otherwise return as a regular identifier. return Token{code: IDENT, literal: buf.String()} } diff --git a/crm/repository/record.go b/crm/repository/record.go index 7071eb2fc..5abbeec05 100644 --- a/crm/repository/record.go +++ b/crm/repository/record.go @@ -6,11 +6,12 @@ import ( "strings" "time" - "github.com/crusttech/crust/crm/repository/ql" "github.com/pkg/errors" "github.com/titpetric/factory" "gopkg.in/Masterminds/squirrel.v1" + "github.com/crusttech/crust/crm/repository/ql" + "github.com/crusttech/crust/crm/types" ) @@ -146,7 +147,6 @@ func (r *record) Find(moduleID uint64, filter string, sort string, page int, per // Create count SQL sentences. count := query.Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")) - sqlSelect, argsSelect, err := count.ToSql() if err != nil { return nil, err