DeforaNetworks/corteza
3
0
Fork 0
Code Activity

pkg/ql refactoring, add proper bool literal handling

Browse Source
This commit is contained in:
Denis Arh
2020-04-30 13:32:59 +02:00
parent efa2f28b8c
commit 9f3a00c0e2
7 changed files with 100 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/ql/ast_nodes.go
View File

@@ -20,14 +20,15 @@ type (
Columns []Column
Null struct{}
LNull struct{}
LBoolean struct{ Value bool }
String struct {
LString struct {
Value string
Args []interface{}
}
Number struct {
LNumber struct {
Value string
}
@@ -60,14 +61,23 @@ type (
}
)
func (n Null) Validate() (err error) { return }
func (n Null) String() string { return "NULL" }
func (n LNull) Validate() (err error) { return }
func (n LNull) String() string { return "NULL" }
func (n String) Validate() (err error) { return }
func (n String) String() string { return fmt.Sprintf("%q", n.Value) }
func (n LBoolean) Validate() (err error) { return }
func (n LBoolean) String() string {
if n.Value {
return "TRUE"
} else {
return "FALSE"
}
}
func (n Number) Validate() (err error) { return }
func (n Number) String() string { return n.Value }
func (n LString) Validate() (err error) { return }
func (n LString) String() string { return fmt.Sprintf("%q", n.Value) }
func (n LNumber) Validate() (err error) { return }
func (n LNumber) String() string { return n.Value }
func (n Operator) Validate() (err error) { return }
func (n Operator) String() string { return n.Kind }

24
pkg/ql/ast_parser.go
View File

@@ -166,8 +166,12 @@ checkToken:
list = append(list, ident)
goto next
}
case NULL:
list = append(list, Null{})
case LNULL:
list = append(list, LNull{})
goto next
case LBOOL:
list = append(list, LBoolean{Value: strings.ToUpper(t.literal) == "TRUE"})
goto next
case OPERATOR:
if len(list) > 0 {
// Merge with previous operator node
@@ -186,11 +190,11 @@ checkToken:
list = append(list, keyword)
}
goto next
case NUMBER:
list = append(list, Number{Value: t.literal})
case LNUMBER:
list = append(list, LNumber{Value: t.literal})
goto next
case STRING:
list = append(list, String{Value: t.literal})
case LSTRING:
list = append(list, LString{Value: t.literal})
goto next
case PARENTHESIS_OPEN:
depth := p.level
@@ -258,11 +262,11 @@ next:
p.level++
parenthesisOpened = true
goto next
case NUMBER:
list = append(list, Number{Value: t.literal})
case LNUMBER:
list = append(list, LNumber{Value: t.literal})
goto next
case STRING:
list = append(list, String{Value: t.literal})
case LSTRING:
list = append(list, LString{Value: t.literal})
goto next
case OPERATOR:
list = append(list, Operator{Kind: t.literal})

54
pkg/ql/ast_parser_test.go
View File

@@ -29,7 +29,7 @@ func TestAstParser_Parser(t *testing.T) {
ASTNodes{
Ident{Value: "arg2"},
Operator{Kind: "/"},
Number{Value: "100"},
LNumber{Value: "100"},
},
},
},
@@ -42,7 +42,7 @@ func TestAstParser_Parser(t *testing.T) {
ASTNodes{
Ident{Value: "arg1"},
Operator{Kind: "*"},
Number{Value: "5"},
LNumber{Value: "5"},
},
},
},
@@ -50,9 +50,9 @@ func TestAstParser_Parser(t *testing.T) {
ASTNodes{
Ident{Value: "arg2"},
Operator{Kind: "/"},
Number{Value: "100"},
LNumber{Value: "100"},
Operator{Kind: "+"},
Number{Value: "10"},
LNumber{Value: "10"},
},
},
},
@@ -63,7 +63,7 @@ func TestAstParser_Parser(t *testing.T) {
Name: "date_format",
Arguments: ASTSet{
Ident{Value: "created_at"},
String{Value: "%Y"},
LString{Value: "%Y"},
},
},
},
@@ -92,7 +92,7 @@ func TestAstParser_Parser(t *testing.T) {
},
},
Operator{Kind: "!="},
Number{Value: "2010"},
LNumber{Value: "2010"},
},
},
{
@@ -106,7 +106,7 @@ func TestAstParser_Parser(t *testing.T) {
},
},
Operator{Kind: "!="},
Number{Value: "2010"},
LNumber{Value: "2010"},
Operator{"AND"},
Function{
Name: "month",
@@ -115,7 +115,7 @@ func TestAstParser_Parser(t *testing.T) {
},
},
Operator{Kind: "="},
Number{Value: "6"},
LNumber{Value: "6"},
},
},
{
@@ -126,7 +126,7 @@ func TestAstParser_Parser(t *testing.T) {
Operator{Kind: "="},
Function{Name: "year", Arguments: ASTSet{Function{Name: "now"}}},
Operator{Kind: "-"},
Number{Value: "1"},
LNumber{Value: "1"},
},
},
{
@@ -144,7 +144,7 @@ func TestAstParser_Parser(t *testing.T) {
tree: ASTNodes{
Ident{Value: "foo"},
Operator{Kind: "LIKE"},
String{Value: "bar%"},
LString{Value: "bar%"},
},
},
{
@@ -153,7 +153,7 @@ func TestAstParser_Parser(t *testing.T) {
tree: ASTNodes{
Ident{Value: "foo"},
Operator{Kind: "NOT LIKE"},
String{Value: "bar%"},
LString{Value: "bar%"},
},
},
{
@@ -162,7 +162,7 @@ func TestAstParser_Parser(t *testing.T) {
tree: ASTNodes{
Ident{Value: "foo"},
Operator{Kind: "="},
Null{},
LNull{},
},
},
{
@@ -171,7 +171,7 @@ func TestAstParser_Parser(t *testing.T) {
tree: ASTNodes{
Ident{Value: "foo"},
Operator{Kind: "IS NOT"},
Null{},
LNull{},
},
},
{
@@ -180,7 +180,7 @@ func TestAstParser_Parser(t *testing.T) {
tree: ASTNodes{
Ident{Value: "foo"},
Operator{Kind: "IS"},
Null{},
LNull{},
},
},
{
@@ -216,6 +216,24 @@ func TestAstParser_Parser(t *testing.T) {
Ident{Value: "foo3"},
},
},
{
parser: NewParser().ParseExpression,
in: `false AND true`,
tree: ASTNodes{
LBoolean{Value: false},
Operator{"AND"},
LBoolean{Value: true},
},
},
{
parser: NewParser().ParseExpression,
in: `false AND true`,
tree: ASTNodes{
LBoolean{Value: false},
Operator{"AND"},
LBoolean{Value: true},
},
},
// @todo support IN (....)
//{
// parser: NewParser().ParseExpression,
@@ -224,9 +242,9 @@ func TestAstParser_Parser(t *testing.T) {
// Ident{Value: "abc"},
// Operator{"IN"},
// ASTNodes{
// Number{Value: "1"},
// Number{Value: "2"},
// Number{Value: "3"},
// LNumber{Value: "1"},
// LNumber{Value: "2"},
// LNumber{Value: "3"},
// },
// },
//},
@@ -333,7 +351,7 @@ func TestAstParser_ColumnParser(t *testing.T) {
Name: "DATE_FORMAT",
Arguments: ASTSet{
Ident{Value: "some_date"},
String{Value: "%Y-%m-01"},
LString{Value: "%Y-%m-01"},
},
},
},

19
pkg/ql/lexer_test.go
View File

@@ -27,10 +27,10 @@ func TestScanner_ScanSimple(t *testing.T) {
{s: `>=`, tok: OPERATOR, lit: ">="},
{s: `<>`, tok: OPERATOR, lit: "<>"},
{s: `+`, tok: OPERATOR, lit: "+"},
{s: `'fooo'`, tok: STRING, lit: "fooo"},
{s: `'escaped \' quote'`, tok: STRING, lit: "escaped ' quote"},
{s: `'double \\ escape'`, tok: STRING, lit: "double \\ escape"},
{s: `12345`, tok: NUMBER, lit: "12345"},
{s: `'fooo'`, tok: LSTRING, lit: "fooo"},
{s: `'escaped \' quote'`, tok: LSTRING, lit: "escaped ' quote"},
{s: `'double \\ escape'`, tok: LSTRING, lit: "double \\ escape"},
{s: `12345`, tok: LNUMBER, lit: "12345"},
// Identifiers
{s: `foo`, tok: IDENT, lit: `foo`},
@@ -39,6 +39,9 @@ func TestScanner_ScanSimple(t *testing.T) {
// Parenthesis
{s: `(`, tok: PARENTHESIS_OPEN, lit: `(`},
{s: `)`, tok: PARENTHESIS_CLOSE, lit: `)`},
// Literals
{s: `true`, tok: LBOOL, lit: `TRUE`},
}
for i, test := range tests {
@@ -63,15 +66,15 @@ func TestScanner_ScanComplex(t *testing.T) {
{`arg1 * arg2`,
[]tokenCode{IDENT, WS, OPERATOR, WS, IDENT}},
{`date_format(created_at,'%Y')`,
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, STRING, PARENTHESIS_CLOSE}},
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, LSTRING, PARENTHESIS_CLOSE}},
{`foo LIKE 'abc%'`,
[]tokenCode{IDENT, WS, OPERATOR, WS, STRING}},
[]tokenCode{IDENT, WS, OPERATOR, WS, LSTRING}},
{`foo NOT LIKE 'abc%'`,
[]tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, STRING}},
[]tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, LSTRING}},
{`foo DESC`,
[]tokenCode{IDENT, WS, KEYWORD}},
{`year(now())-1`,
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, PARENTHESIS_OPEN, PARENTHESIS_CLOSE, PARENTHESIS_CLOSE, OPERATOR, NUMBER}},
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, PARENTHESIS_OPEN, PARENTHESIS_CLOSE, PARENTHESIS_CLOSE, OPERATOR, LNUMBER}},
}
for _, test := range tests {

14
pkg/ql/squirrel.go
View File

@@ -68,10 +68,18 @@ func (n Ident) ToSql() (string, []interface{}, error) {
return n.Value, n.Args, nil
}
func (n Null) ToSql() (string, []interface{}, error) {
func (n LNull) ToSql() (string, []interface{}, error) {
return "NULL", nil, nil
}
func (n LBoolean) ToSql() (string, []interface{}, error) {
if n.Value {
return "TRUE", nil, nil
} else {
return "FALSE", nil, nil
}
}
func (n Function) ToSql() (string, []interface{}, error) {
if paramsSql, args, err := n.Arguments.ToSql(); err != nil {
return "", nil, err
@@ -101,10 +109,10 @@ func (n Operator) ToSql() (string, []interface{}, error) {
return " " + op + " ", nil, nil
}
func (n String) ToSql() (string, []interface{}, error) {
func (n LString) ToSql() (string, []interface{}, error) {
return "?", []interface{}{n.Value}, nil
}
func (n Number) ToSql() (string, []interface{}, error) {
func (n LNumber) ToSql() (string, []interface{}, error) {
return n.Value, nil, nil
}

7
pkg/ql/token_codes.go
View File

@@ -16,13 +16,14 @@ const (
EOF
WS // 2
IDENT
NUMBER // 4
STRING
LNULL
LBOOL // 4
LNUMBER
LSTRING
COMMA // ,
DOT // .
OPERATOR // + - / *
PARENTHESIS_OPEN
PARENTHESIS_CLOSE
KEYWORD
NULL
)

8
pkg/ql/token_consumers.go
View File

@@ -86,7 +86,9 @@ func (TokenConsumerIdent) Consume(s RuneReader) Token {
switch lit {
case "NULL":
return Token{code: NULL}
return Token{code: LNULL}
case "TRUE", "FALSE":
return Token{code: LBOOL, literal: lit}
case "IS", "LIKE", "NOT", "AND", "OR", "XOR":
return Token{code: OPERATOR, literal: lit}
case "DESC", "ASC", "INTERVAL":
@@ -111,7 +113,7 @@ func (str TokenConsumerString) Consume(s RuneReader) Token {
if ch = s.read(); ch == eof {
break
} else if !escaping && str.Test(ch) { // test for quote
return Token{code: STRING, literal: buf.String()}
return Token{code: LSTRING, literal: buf.String()}
} else {
escaping = !escaping && ch == '\\'
if !escaping {
@@ -147,7 +149,7 @@ func (str TokenConsumerNumber) Consume(s RuneReader) Token {
}
// Otherwise return as a regular identifier.
return Token{code: NUMBER, literal: buf.String()}
return Token{code: LNUMBER, literal: buf.String()}
}
// isLetter returns true if the rune is a letter.