Add support for new operators and sorting keywords

crm/repository/ql/ast_parser.go

@@ -162,12 +162,22 @@ checkToken:
 			return nil, err
 		} else {
 			list = append(list, ident)
-
 			goto next
 		}
 	case OPERATOR:
+		if len(list) > 0 {
+			// Merge with previous operator node
+			if prevOp, ok := list[len(list)-1].(Operator); ok {
+				list[len(list)-1] = Operator{Kind: prevOp.Kind + " " + t.literal}
+				goto next
+			}
+		}
+
 		list = append(list, Operator{Kind: t.literal})
 		goto next
+	case KEYWORD:
+		list = append(list, Keyword{Keyword: t.literal})
+		goto next
 	case NUMBER:
 		list = append(list, Number{Value: t.literal})
 		goto next
@@ -189,11 +199,7 @@ checkToken:
 }
 
 func (p *Parser) parseIdent(t Token) (list ASTNode, err error) {
-	var ucLiteral = strings.ToUpper(t.literal)
-
-	if ucLiteral == "AND" || ucLiteral == "OR" {
-		return Keyword{ucLiteral}, nil
-	} else if p.peekToken(1).Is(PARENTHESIS_OPEN) {
+	if p.peekToken(1).Is(PARENTHESIS_OPEN) {
 		// Handle function calls: <IDENT><PARENTHESIS_OPEN>...
 		f := Function{Name: t.literal}
 		if f.Arguments, err = p.parseSet(); err != nil {

crm/repository/ql/ast_parser_test.go

@@ -151,7 +151,7 @@ func TestAstParser_ParseExpression(t *testing.T) {
 				},
 				Operator{Kind: "!="},
 				Number{Value: "2010"},
-				Keyword{"AND"},
+				Operator{"AND"},
 				Function{
 					Name: "month",
 					Arguments: ASTSet{
@@ -162,6 +162,22 @@ func TestAstParser_ParseExpression(t *testing.T) {
 				Number{Value: "6"},
 			},
 		},
+		{
+			in: `foo LIKE 'bar%'`,
+			tree: ASTNodes{
+				Ident{Value: "foo"},
+				Operator{Kind: "LIKE"},
+				String{Value: "bar%"},
+			},
+		},
+		{
+			in: `foo NOT LIKE 'bar%'`,
+			tree: ASTNodes{
+				Ident{Value: "foo"},
+				Operator{Kind: "NOT LIKE"},
+				String{Value: "bar%"},
+			},
+		},
 	}
 
 	for i, test := range tests {
@@ -210,6 +226,22 @@ func TestAstParser_ColumnParser(t *testing.T) {
 				},
 			},
 		},
+		{
+			in: `a DESC`,
+			cols: Columns{
+				Column{
+					Expr: ASTNodes{Ident{Value: "a"}, Keyword{Keyword: "DESC"}},
+				},
+			},
+		},
+		{
+			in: `a ASC`,
+			cols: Columns{
+				Column{
+					Expr: ASTNodes{Ident{Value: "a"}, Keyword{Keyword: "ASC"}},
+				},
+			},
+		},
 	}
 
 	for i, test := range tests {

crm/repository/ql/lexer_test.go

@@ -58,9 +58,18 @@ func TestScanner_ScanComplex(t *testing.T) {
 		tokens []tokenCode
 	}{
 		// Special tokens (EOF, ILLEGAL, WS)
-		{`func(arg1, arg2)`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}},
-		{`arg1 * arg2`, []tokenCode{IDENT, WS, OPERATOR, WS, IDENT}},
-		{`date_format(created_at,'%Y')`, []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}},
+		{`func(arg1, arg2)`,
+			[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}},
+		{`arg1 * arg2`,
+			[]tokenCode{IDENT, WS, OPERATOR, WS, IDENT}},
+		{`date_format(created_at,'%Y')`,
+			[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}},
+		{`foo LIKE 'abc%'`,
+			[]tokenCode{IDENT, WS, OPERATOR, WS, STRING}},
+		{`foo NOT LIKE 'abc%'`,
+			[]tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, STRING}},
+		{`foo DESC`,
+			[]tokenCode{IDENT, WS, KEYWORD}},
 	}
 
 	for _, test := range tests {

crm/repository/ql/squirrel.go

@@ -76,7 +76,15 @@ func (n Keyword) ToSql() (string, []interface{}, error) {
 }
 
 func (n Operator) ToSql() (string, []interface{}, error) {
-	return n.Kind, nil, nil
+	var op = n.Kind
+
+	switch n.Kind {
+	case "LIKE", "NOT LIKE":
+		// Make sure we are doing case insensitive search
+		op = "COLLATE utf8mb4_general_ci " + n.Kind
+	}
+
+	return " " + op + " ", nil, nil
 }
 
 func (n String) ToSql() (string, []interface{}, error) {

crm/repository/ql/token_codes.go

@@ -22,4 +22,5 @@ const (
 	OPERATOR // + - / *
 	PARENTHESIS_OPEN
 	PARENTHESIS_CLOSE
+	KEYWORD
 )

crm/repository/ql/token_consumers.go

@@ -2,6 +2,7 @@ package ql
 
 import (
 	"bytes"
+	"strings"
 )
 
 type (
@@ -75,6 +76,15 @@ func (TokenConsumerIdent) Consume(s RuneReader) Token {
 		}
 	}
 
+	lit := strings.ToUpper(buf.String())
+
+	switch lit {
+	case "LIKE", "NOT", "AND", "OR", "XOR":
+		return Token{code: OPERATOR, literal: lit}
+	case "DESC", "ASC":
+		return Token{code: KEYWORD, literal: lit}
+	}
+
 	// Otherwise return as a regular identifier.
 	return Token{code: IDENT, literal: buf.String()}
 }

crm/repository/record.go

@@ -6,11 +6,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/crusttech/crust/crm/repository/ql"
 	"github.com/pkg/errors"
 	"github.com/titpetric/factory"
 	"gopkg.in/Masterminds/squirrel.v1"
 
+	"github.com/crusttech/crust/crm/repository/ql"
+
 	"github.com/crusttech/crust/crm/types"
 )
 
@@ -146,7 +147,6 @@ func (r *record) Find(moduleID uint64, filter string, sort string, page int, per
 
 	// Create count SQL sentences.
 	count := query.Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count"))
-
 	sqlSelect, argsSelect, err := count.ToSql()
 	if err != nil {
 		return nil, err