Refactor pkg/ql for more flexibility
Created a temporary pkg/qlng package which contains the reworked logic to avoid (potentially) breaking the rest of the system. Most of the testing will be done implicitly via reporting tests.
This commit is contained in:
Tomaž Jerman
@@ -0,0 +1,231 @@
|
||||
package qlng
|
||||
|
||||
type (
|
||||
ASTNode struct {
|
||||
pMeta *parserMeta
|
||||
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Args ASTNodeSet `json:"args,omitempty"`
|
||||
|
||||
Symbol string `json:"symbol,omitempty"`
|
||||
Value *typedValueWrap `json:"value,omitempty"`
|
||||
|
||||
Raw string `json:"raw,omitempty"`
|
||||
}
|
||||
ASTNodeSet []*ASTNode
|
||||
typedValueWrap struct {
|
||||
Value interface{} `json:"@value"`
|
||||
Type string `json:"@type"`
|
||||
}
|
||||
|
||||
parserMeta struct {
|
||||
opDef *opDef
|
||||
}
|
||||
)
|
||||
|
||||
func MakeValueOf(t string, v interface{}) *typedValueWrap {
|
||||
return &typedValueWrap{
|
||||
Type: t,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
// Traverse traverses the AST down to leaf nodes.
|
||||
//
|
||||
// If fnc. returns false, the traversal of the current branch ends.
|
||||
func (n *ASTNode) Traverse(f func(*ASTNode) (bool, *ASTNode, error)) (err error) {
|
||||
var ok bool
|
||||
var r *ASTNode
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
ok, r, err = f(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*n = *r
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
for _, a := range n.Args {
|
||||
if err = a.Traverse(f); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (n ASTNode) Clone() *ASTNode {
|
||||
aa := n.Args
|
||||
|
||||
if n.Args != nil {
|
||||
n.Args = make(ASTNodeSet, len(aa))
|
||||
for i, a := range aa {
|
||||
n.Args[i] = a.Clone()
|
||||
}
|
||||
}
|
||||
|
||||
if n.Value != nil {
|
||||
n.Value = &typedValueWrap{
|
||||
Type: n.Value.Type,
|
||||
Value: n.Value.Value,
|
||||
}
|
||||
}
|
||||
|
||||
return &n
|
||||
}
|
||||
|
||||
func (n lNull) ToAST() (out *ASTNode) {
|
||||
return &ASTNode{
|
||||
Ref: "null",
|
||||
}
|
||||
}
|
||||
|
||||
func (n lBoolean) ToAST() (out *ASTNode) {
|
||||
return &ASTNode{
|
||||
Value: MakeValueOf("Boolean", n.value),
|
||||
}
|
||||
}
|
||||
|
||||
func (n lString) ToAST() (out *ASTNode) {
|
||||
return &ASTNode{
|
||||
Value: MakeValueOf("String", n.value),
|
||||
}
|
||||
}
|
||||
|
||||
// @todo differentiate between floats and others
|
||||
func (n lNumber) ToAST() (out *ASTNode) {
|
||||
if isFloaty(n.value) {
|
||||
return &ASTNode{
|
||||
Value: MakeValueOf("Float", n.value),
|
||||
}
|
||||
} else {
|
||||
return &ASTNode{
|
||||
Value: MakeValueOf("Int", n.value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (n operator) ToAST() (out *ASTNode) {
|
||||
op := getOp(n.kind)
|
||||
|
||||
return &ASTNode{
|
||||
Ref: op.name,
|
||||
Args: make(ASTNodeSet, 0, 2),
|
||||
pMeta: &parserMeta{opDef: op},
|
||||
}
|
||||
}
|
||||
|
||||
func (n Ident) ToAST() (out *ASTNode) {
|
||||
return &ASTNode{
|
||||
Symbol: n.Value,
|
||||
}
|
||||
}
|
||||
|
||||
func (n keyword) ToAST() (out *ASTNode) {
|
||||
return &ASTNode{
|
||||
Ref: n.keyword,
|
||||
}
|
||||
}
|
||||
|
||||
func (n interval) ToAST() (out *ASTNode) {
|
||||
return &ASTNode{
|
||||
Ref: "interval",
|
||||
Args: ASTNodeSet{
|
||||
{Symbol: n.unit},
|
||||
{Value: MakeValueOf("Number", n.value)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (n function) ToAST() (out *ASTNode) {
|
||||
auxA := n.arguments.ToAST()
|
||||
|
||||
return &ASTNode{
|
||||
Ref: n.name,
|
||||
Args: auxA.Args,
|
||||
}
|
||||
}
|
||||
|
||||
func (nn parserNodeSet) ToAST() (out *ASTNode) {
|
||||
auxArgs := make(ASTNodeSet, 0, len(nn))
|
||||
|
||||
for _, n := range nn {
|
||||
auxArgs = append(auxArgs, n.ToAST())
|
||||
}
|
||||
|
||||
return &ASTNode{
|
||||
Ref: "group",
|
||||
Args: auxArgs,
|
||||
}
|
||||
}
|
||||
|
||||
func (nn parserNodes) ToAST() (out *ASTNode) {
|
||||
// Prep
|
||||
auxArgs := make(ASTNodeSet, 0, len(nn))
|
||||
|
||||
// Convert the entire level to AST nodes
|
||||
for _, n := range nn {
|
||||
auxArgs = append(auxArgs, n.ToAST())
|
||||
}
|
||||
|
||||
// In the current level, have the operators consume operands
|
||||
// based on their defined weight
|
||||
//
|
||||
// - find the highest prio. op and have it consume what it wants
|
||||
// - repeat until all ops are satisfied
|
||||
// -- post optimizations?
|
||||
for {
|
||||
var bestOp *opDef
|
||||
bestOpIx := -1
|
||||
|
||||
// We're done when it is reduced to 1
|
||||
if len(auxArgs) <= 1 {
|
||||
break
|
||||
}
|
||||
|
||||
for _i, _a := range auxArgs {
|
||||
i := _i
|
||||
a := _a
|
||||
|
||||
// use this as a delimiter for now
|
||||
if a.pMeta == nil || a.pMeta.opDef == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if bestOp == nil {
|
||||
bestOp = a.pMeta.opDef
|
||||
bestOpIx = i
|
||||
continue
|
||||
}
|
||||
|
||||
if a.pMeta.opDef.weight < bestOp.weight {
|
||||
bestOp = a.pMeta.opDef
|
||||
bestOpIx = i
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Have the op consume what it needs.
|
||||
// Currently we only have binary operators (the !unary ones) so this is fine.
|
||||
arg := auxArgs[bestOpIx]
|
||||
arg.Args = append(arg.Args, auxArgs[bestOpIx-1], auxArgs[bestOpIx+1])
|
||||
// this is not needed anymore so we can remove it
|
||||
arg.pMeta = nil
|
||||
|
||||
// Remove the consumed bits and replace it with the new bit
|
||||
aux := auxArgs[0 : bestOpIx-1]
|
||||
aux = append(aux, arg)
|
||||
// +1 for right side, +1 because the left index is inclusive
|
||||
aux = append(aux, auxArgs[bestOpIx+2:]...)
|
||||
auxArgs = aux
|
||||
}
|
||||
|
||||
return &ASTNode{
|
||||
Ref: "group",
|
||||
Args: auxArgs,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Ensure the parser can parse strings into Statement ASTs.
|
||||
func Test_Validators(t *testing.T) {
|
||||
var tests = []struct {
|
||||
tree parserNode
|
||||
}{
|
||||
{
|
||||
tree: parserNodes{
|
||||
Ident{Value: "foo"},
|
||||
operator{kind: "="},
|
||||
},
|
||||
},
|
||||
{
|
||||
tree: parserNodes{
|
||||
operator{kind: "="},
|
||||
Ident{Value: "foo"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
if err := test.tree.Validate(); err == nil {
|
||||
t.Fatalf("expecting error, got nil:\n"+
|
||||
" test case: %d. %q", i, test.tree.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Lexer represents a lexical scanner.
|
||||
type (
|
||||
RuneReader interface {
|
||||
read() rune
|
||||
unread()
|
||||
}
|
||||
|
||||
TokenConsumers interface {
|
||||
Test(ch rune) bool
|
||||
Consume(s RuneReader) Token
|
||||
}
|
||||
|
||||
Token struct {
|
||||
code tokenCode
|
||||
literal string
|
||||
line uint
|
||||
char uint
|
||||
}
|
||||
|
||||
Lexer struct {
|
||||
r *bufio.Reader
|
||||
consumers []TokenConsumers
|
||||
line uint
|
||||
char uint
|
||||
}
|
||||
)
|
||||
|
||||
// eof represents a marker rune for the end of the reader.
|
||||
var eof = rune(0)
|
||||
|
||||
// NewLexer returns a new instance of Lexer.
|
||||
func NewLexer(r io.Reader) *Lexer {
|
||||
return &Lexer{
|
||||
r: bufio.NewReader(r),
|
||||
consumers: []TokenConsumers{
|
||||
&TokenConsumerGeneric{token: WS, whitelist: CHAR_WHITELIST_WHITESPACE},
|
||||
// @todo ensure operator order (eg != is valid, =! is not)
|
||||
&TokenConsumerGeneric{token: OPERATOR, whitelist: CHAR_WHITELIST_OPERATORS},
|
||||
&TokenConsumerGeneric{token: COMMA, whitelist: ",", maxLength: 1},
|
||||
&TokenConsumerGeneric{token: DOT, whitelist: ".", maxLength: 1},
|
||||
&TokenConsumerGeneric{token: PARENTHESIS_OPEN, whitelist: "(", maxLength: 1},
|
||||
&TokenConsumerGeneric{token: PARENTHESIS_CLOSE, whitelist: ")", maxLength: 1},
|
||||
&TokenConsumerString{},
|
||||
&TokenConsumerNumber{},
|
||||
&TokenConsumerIdent{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Scan returns the next token and literal value.
|
||||
func (s *Lexer) Scan() Token {
|
||||
var ch = s.peek()
|
||||
|
||||
if ch == '\n' {
|
||||
s.line++
|
||||
s.char = 0
|
||||
}
|
||||
|
||||
if eof == ch {
|
||||
return Token{code: EOF, line: s.line, char: s.char}
|
||||
}
|
||||
|
||||
for _, c := range s.consumers {
|
||||
if c.Test(ch) {
|
||||
t := c.Consume(s)
|
||||
t.line = s.line
|
||||
t.char = s.char
|
||||
return t
|
||||
}
|
||||
}
|
||||
|
||||
return Token{code: ILLEGAL, literal: string(ch), line: s.line, char: s.char}
|
||||
}
|
||||
|
||||
// read reads the next rune from the buffered reader.
|
||||
// Returns the rune(0) if an error occurs (or io.EOF is returned).
|
||||
func (s *Lexer) read() rune {
|
||||
ch, _, err := s.r.ReadRune()
|
||||
s.char++
|
||||
if err != nil {
|
||||
return eof
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
func (s *Lexer) peek() rune {
|
||||
bb, err := s.r.Peek(1)
|
||||
if err != nil || len(bb) == 0 {
|
||||
return eof
|
||||
}
|
||||
return rune(bb[0])
|
||||
}
|
||||
|
||||
// unread places the previously read rune back on the reader.
|
||||
func (s *Lexer) unread() { _ = s.r.UnreadRune() }
|
||||
|
||||
func (t Token) Is(cc ...tokenCode) bool {
|
||||
for _, c := range cc {
|
||||
if t.code == c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Ensure the scanner can scan tokens correctly.
|
||||
func TestScanner_ScanSimple(t *testing.T) {
|
||||
var tests = []struct {
|
||||
s string
|
||||
tok tokenCode
|
||||
lit string
|
||||
}{
|
||||
// Special tokens (EOF, ILLEGAL, WS)
|
||||
{s: ``, tok: EOF},
|
||||
{s: `#`, tok: ILLEGAL, lit: `#`},
|
||||
{s: ` `, tok: WS, lit: " "},
|
||||
{s: "\t", tok: WS, lit: "\t"},
|
||||
{s: "\n", tok: WS, lit: "\n"},
|
||||
|
||||
// Operators
|
||||
{s: `*`, tok: OPERATOR, lit: "*"},
|
||||
{s: `!=`, tok: OPERATOR, lit: "!="},
|
||||
{s: `<`, tok: OPERATOR, lit: "<"},
|
||||
{s: `>`, tok: OPERATOR, lit: ">"},
|
||||
{s: `>=`, tok: OPERATOR, lit: ">="},
|
||||
{s: `<>`, tok: OPERATOR, lit: "<>"},
|
||||
{s: `+`, tok: OPERATOR, lit: "+"},
|
||||
{s: `'fooo'`, tok: LSTRING, lit: "fooo"},
|
||||
{s: `'escaped \' quote'`, tok: LSTRING, lit: "escaped ' quote"},
|
||||
{s: `'double \\ escape'`, tok: LSTRING, lit: "double \\ escape"},
|
||||
{s: `12345`, tok: LNUMBER, lit: "12345"},
|
||||
|
||||
// Identifiers
|
||||
{s: `foo`, tok: IDENT, lit: `foo`},
|
||||
{s: `Zx12_3U_-`, tok: IDENT, lit: `Zx12_3U_`},
|
||||
|
||||
// Parenthesis
|
||||
{s: `(`, tok: PARENTHESIS_OPEN, lit: `(`},
|
||||
{s: `)`, tok: PARENTHESIS_CLOSE, lit: `)`},
|
||||
|
||||
// Literals
|
||||
{s: `true`, tok: LBOOL, lit: `TRUE`},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
s := NewLexer(strings.NewReader(test.s))
|
||||
tok := s.Scan()
|
||||
if test.tok != tok.code {
|
||||
t.Errorf("%d. %q token mismatch: exp=%d got=%d <%q>", i, test.s, test.tok, tok.code, tok.literal)
|
||||
} else if test.lit != tok.literal {
|
||||
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, test.s, test.lit, tok.literal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanner_ScanComplex(t *testing.T) {
|
||||
var tests = []struct {
|
||||
s string
|
||||
tokens []tokenCode
|
||||
}{
|
||||
// Special tokens (EOF, ILLEGAL, WS)
|
||||
{`func(arg1, arg2)`,
|
||||
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}},
|
||||
{`arg1 * arg2`,
|
||||
[]tokenCode{IDENT, WS, OPERATOR, WS, IDENT}},
|
||||
{`date_format(created_at,'%Y')`,
|
||||
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, LSTRING, PARENTHESIS_CLOSE}},
|
||||
{`foo LIKE 'abc%'`,
|
||||
[]tokenCode{IDENT, WS, OPERATOR, WS, LSTRING}},
|
||||
{`foo NOT LIKE 'abc%'`,
|
||||
[]tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, LSTRING}},
|
||||
{`foo DESC`,
|
||||
[]tokenCode{IDENT, WS, KEYWORD}},
|
||||
{`year(now())-1`,
|
||||
[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, PARENTHESIS_OPEN, PARENTHESIS_CLOSE, PARENTHESIS_CLOSE, OPERATOR, LNUMBER}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
var tokens []Token
|
||||
s := NewLexer(strings.NewReader(test.s))
|
||||
for {
|
||||
tok := s.Scan()
|
||||
if tok.Is(EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
tokens = append(tokens, tok)
|
||||
}
|
||||
|
||||
if len(tokens) != len(test.tokens) {
|
||||
t.Errorf("Collected tokens do not match (%v)", tokens)
|
||||
}
|
||||
|
||||
for i := 0; i < len(tokens); i++ {
|
||||
if tokens[i].code != test.tokens[i] {
|
||||
t.Errorf("Input: %s", test.s)
|
||||
t.Errorf("Expected: %v", test.tokens)
|
||||
t.Errorf("Collected: %v", tokens)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type (
|
||||
// Parser represents a parser.
|
||||
Parser struct {
|
||||
lexer *Lexer
|
||||
tokbuf []Token
|
||||
|
||||
OnIdent IdentHandler
|
||||
OnFunction FunctionHandler
|
||||
|
||||
// parenthesis level control
|
||||
level uint
|
||||
}
|
||||
|
||||
IdentHandler func(ident Ident) (Ident, error)
|
||||
FunctionHandler func(ident function) (parserNode, error)
|
||||
)
|
||||
|
||||
// NewParser returns a new instance of Parser.
|
||||
func NewParser() *Parser {
|
||||
p := &Parser{
|
||||
OnIdent: func(ident Ident) (Ident, error) { return ident, nil },
|
||||
OnFunction: func(ident function) (parserNode, error) { return ident, nil },
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// Removes oldest token in the buffer, adds new one and returns 2nd oldest
|
||||
func (p *Parser) nextToken() Token {
|
||||
var t Token
|
||||
for {
|
||||
t = p.lexer.Scan()
|
||||
if !t.Is(WS) {
|
||||
p.tokbuf = append(p.tokbuf[1:], t)
|
||||
return p.tokbuf[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) peekToken(s int) Token {
|
||||
return p.tokbuf[s]
|
||||
}
|
||||
|
||||
func (p *Parser) initLexer(s string) {
|
||||
p.lexer = NewLexer(strings.NewReader(s))
|
||||
p.tokbuf = make([]Token, 3)
|
||||
|
||||
for c := 1; c < cap(p.tokbuf); c++ {
|
||||
// Fill the buffer
|
||||
p.nextToken()
|
||||
}
|
||||
}
|
||||
|
||||
// Parse parses the given expression and returns the generated AST
|
||||
func (p *Parser) Parse(s string) (*ASTNode, error) {
|
||||
p.initLexer(s)
|
||||
|
||||
if set, err := p.parse(p.nextToken()); err != nil {
|
||||
return nil, err
|
||||
} else if len(set) == 1 {
|
||||
return set[0].ToAST(), set[0].Validate()
|
||||
} else {
|
||||
return set.ToAST(), set.Validate()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Peek ahead if there is an alias ident (<IDENT:AS>)
|
||||
//
|
||||
// @todo remove this; not used anywhere after report builder gets removed
|
||||
func (p *Parser) peekIfAlias() bool {
|
||||
var f, s = p.peekToken(1), p.peekToken(2)
|
||||
return f.Is(IDENT) && strings.ToUpper(f.literal) == "AS" && s.Is(IDENT)
|
||||
}
|
||||
|
||||
func (p *Parser) parse(t Token) (list parserNodes, err error) {
|
||||
goto checkToken
|
||||
|
||||
next:
|
||||
if p.peekToken(1).Is(COMMA) || p.peekIfAlias() {
|
||||
// Peek ahead and exit on comma
|
||||
return
|
||||
}
|
||||
if p.peekToken(1).Is(PARENTHESIS_CLOSE) {
|
||||
// Peek ahead and exit on closed parenthesis
|
||||
if p.level == 0 {
|
||||
return nil, fmt.Errorf("closing unopened parenthesis in expression")
|
||||
}
|
||||
p.level--
|
||||
return
|
||||
}
|
||||
|
||||
t = p.nextToken()
|
||||
|
||||
checkToken:
|
||||
switch t.code {
|
||||
case EOF:
|
||||
break
|
||||
case WS:
|
||||
// Ignore ws... next token...
|
||||
goto next
|
||||
case ILLEGAL:
|
||||
return nil, fmt.Errorf("found an illegal token (%+v)", t)
|
||||
case IDENT:
|
||||
var ident parserNode
|
||||
if ident, err = p.parseIdent(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, ident)
|
||||
goto next
|
||||
}
|
||||
case LNULL:
|
||||
list = append(list, lNull{})
|
||||
goto next
|
||||
case LBOOL:
|
||||
list = append(list, lBoolean{value: evalBool(t.literal)})
|
||||
goto next
|
||||
case OPERATOR:
|
||||
if len(list) > 0 {
|
||||
// Merge with previous operator node
|
||||
if prevOp, ok := list[len(list)-1].(operator); ok {
|
||||
list[len(list)-1] = operator{kind: prevOp.kind + " " + t.literal}
|
||||
goto next
|
||||
}
|
||||
}
|
||||
|
||||
list = append(list, operator{kind: t.literal})
|
||||
goto next
|
||||
case KEYWORD:
|
||||
if keyword, err := p.parseKeyword(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, keyword)
|
||||
}
|
||||
goto next
|
||||
case LNUMBER:
|
||||
list = append(list, lNumber{value: t.literal})
|
||||
goto next
|
||||
case LSTRING:
|
||||
list = append(list, lString{value: t.literal})
|
||||
goto next
|
||||
case PARENTHESIS_OPEN:
|
||||
depth := p.level
|
||||
p.level++
|
||||
if sub, err := p.parse(p.nextToken()); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, sub)
|
||||
}
|
||||
|
||||
// Allow parent level to continue parsing.
|
||||
// Example: ((A) AND (B))
|
||||
// +1 since PARENTHESIS_CLOSE decrease level
|
||||
if (p.level + 1) != depth {
|
||||
p.nextToken()
|
||||
goto next
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected token while parsing expression (%v)", t)
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (p *Parser) parseIdent(t Token) (list parserNode, err error) {
|
||||
if p.peekToken(1).Is(PARENTHESIS_OPEN) {
|
||||
// Handle function calls: <IDENT><PARENTHESIS_OPEN>...
|
||||
f := function{name: t.literal}
|
||||
if f.arguments, err = p.parseSet(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return p.OnFunction(f)
|
||||
}
|
||||
}
|
||||
|
||||
i := Ident{Value: t.literal}
|
||||
|
||||
if p.peekToken(1).Is(DOT) {
|
||||
p.nextToken()
|
||||
i.Value += "."
|
||||
l2 := p.nextToken()
|
||||
if l2.Is(IDENT) {
|
||||
i.Value += l2.literal
|
||||
}
|
||||
}
|
||||
|
||||
return p.OnIdent(i)
|
||||
}
|
||||
|
||||
func (p *Parser) parseSet() (list parserNodeSet, err error) {
|
||||
var expr parserNodes
|
||||
var parenthesisOpened = false
|
||||
|
||||
next:
|
||||
t := p.nextToken()
|
||||
|
||||
if p.peekIfAlias() {
|
||||
return
|
||||
}
|
||||
|
||||
switch t.code {
|
||||
case WS:
|
||||
goto next
|
||||
case PARENTHESIS_OPEN:
|
||||
p.level++
|
||||
parenthesisOpened = true
|
||||
goto next
|
||||
case LNUMBER:
|
||||
list = append(list, lNumber{value: t.literal})
|
||||
goto next
|
||||
case LSTRING:
|
||||
list = append(list, lString{value: t.literal})
|
||||
goto next
|
||||
case OPERATOR:
|
||||
list = append(list, operator{kind: t.literal})
|
||||
goto next
|
||||
case KEYWORD:
|
||||
if keyword, err := p.parseKeyword(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, keyword)
|
||||
}
|
||||
goto next
|
||||
case IDENT:
|
||||
if p.peekToken(1).Is(OPERATOR) {
|
||||
// Looks like we have an expression ahead of us
|
||||
|
||||
if parenthesisOpened {
|
||||
// Expression will find closing parenthesis and dec. the level
|
||||
// so, lets bump up the number
|
||||
p.level++
|
||||
}
|
||||
|
||||
if expr, err = p.parse(t); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
list = append(list, expr)
|
||||
goto next
|
||||
}
|
||||
|
||||
var ident parserNode
|
||||
if ident, err = p.parseIdent(t); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
list = append(list, ident)
|
||||
|
||||
goto next
|
||||
}
|
||||
case COMMA:
|
||||
goto next
|
||||
case EOF:
|
||||
return
|
||||
case PARENTHESIS_CLOSE:
|
||||
// Peek ahead and exit on closed parenthesis
|
||||
if p.level == 0 {
|
||||
return nil, fmt.Errorf("closing unopened parenthesis in set")
|
||||
}
|
||||
p.level--
|
||||
return
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected token while parsing set (%v)", t)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) parseKeyword(t Token) (list parserNode, err error) {
|
||||
switch strings.ToUpper(t.literal) {
|
||||
case "INTERVAL":
|
||||
i := interval{value: p.nextToken().literal}
|
||||
u := p.nextToken()
|
||||
|
||||
if u.code != IDENT {
|
||||
return nil, fmt.Errorf("expecting identifier, got %v", t)
|
||||
} else {
|
||||
switch strings.ToUpper(u.literal) {
|
||||
case "MICROSECOND", "SECOND", "MINUTE", "HOUR",
|
||||
"DAY", "WEEK", "MONTH", "QUARTER", "YEAR",
|
||||
"SECOND_MICROSECOND", "MINUTE_MICROSECOND", "MINUTE_SECOND", "HOUR_MICROSECOND", "HOUR_SECOND",
|
||||
"HOUR_MINUTE", "DAY_MICROSECOND", "DAY_SECOND", "DAY_MINUTE", "DAY_HOUR", "YEAR_MONTH":
|
||||
// All good
|
||||
break
|
||||
default:
|
||||
return nil, fmt.Errorf("expecting interval unit, got %v", u.literal)
|
||||
}
|
||||
}
|
||||
|
||||
i.unit = u.literal
|
||||
|
||||
return i, nil
|
||||
|
||||
default:
|
||||
return keyword{keyword: t.literal}, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SelectStatement represents a SQL SELECT statement.
|
||||
type (
|
||||
parserNode interface {
|
||||
fmt.Stringer
|
||||
|
||||
Validate() error
|
||||
ToAST() *ASTNode
|
||||
}
|
||||
|
||||
parserNodeSet []parserNode // Stream of comma delimited nodes
|
||||
parserNodes []parserNode // Stream of space delimited nodes
|
||||
|
||||
lNull struct{}
|
||||
lBoolean struct{ value bool }
|
||||
|
||||
lString struct {
|
||||
value string
|
||||
args []interface{}
|
||||
}
|
||||
|
||||
lNumber struct {
|
||||
value string
|
||||
}
|
||||
|
||||
operator struct {
|
||||
kind string
|
||||
}
|
||||
|
||||
Ident struct {
|
||||
Value string
|
||||
args []interface{}
|
||||
}
|
||||
|
||||
keyword struct {
|
||||
keyword string
|
||||
}
|
||||
|
||||
interval struct {
|
||||
value string
|
||||
unit string
|
||||
}
|
||||
|
||||
function struct {
|
||||
name string
|
||||
arguments parserNodeSet
|
||||
}
|
||||
|
||||
opDef struct {
|
||||
name string
|
||||
weight int
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
ops = map[string]opDef{
|
||||
// generic comparison
|
||||
`=`: {name: `eq`, weight: 40},
|
||||
`==`: {name: `eq`, weight: 40},
|
||||
`===`: {name: `eq`, weight: 40},
|
||||
`!=`: {name: `ne`, weight: 40},
|
||||
`!==`: {name: `ne`, weight: 40},
|
||||
`<>`: {name: `ne`, weight: 40},
|
||||
`<`: {name: `lt`, weight: 30},
|
||||
`<=`: {name: `le`, weight: 30},
|
||||
`>`: {name: `gt`, weight: 30},
|
||||
`>=`: {name: `ge`, weight: 30},
|
||||
`IS`: {name: `is`, weight: 40},
|
||||
`IS NOT`: {name: `nis`, weight: 40},
|
||||
|
||||
// conjunction
|
||||
`AND`: {name: `and`, weight: 50},
|
||||
`&&`: {name: `and`, weight: 50},
|
||||
`OR`: {name: `or`, weight: 60},
|
||||
`||`: {name: `or`, weight: 60},
|
||||
`XOR`: {name: `xor`, weight: 60},
|
||||
|
||||
// math
|
||||
`+`: {name: `add`, weight: 20},
|
||||
`-`: {name: `sub`, weight: 20},
|
||||
`*`: {name: `mult`, weight: 10},
|
||||
`/`: {name: `div`, weight: 10},
|
||||
|
||||
// str comp.
|
||||
`LIKE`: {name: `like`, weight: 40},
|
||||
`NOT LIKE`: {name: `nlike`, weight: 40},
|
||||
}
|
||||
)
|
||||
|
||||
func getOp(op string) *opDef {
|
||||
o, ok := ops[strings.ToUpper(op)]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return &o
|
||||
}
|
||||
|
||||
func (n lNull) Validate() (err error) { return }
|
||||
func (n lNull) String() string { return "NULL" }
|
||||
|
||||
func (n lBoolean) Validate() (err error) { return }
|
||||
func (n lBoolean) String() string {
|
||||
if n.value {
|
||||
return "TRUE"
|
||||
} else {
|
||||
return "FALSE"
|
||||
}
|
||||
}
|
||||
|
||||
func (n lString) Validate() (err error) { return }
|
||||
func (n lString) String() string { return fmt.Sprintf("%q", n.value) }
|
||||
|
||||
func (n lNumber) Validate() (err error) { return }
|
||||
func (n lNumber) String() string { return n.value }
|
||||
|
||||
func (n operator) Validate() (err error) {
|
||||
if getOp(n.kind) == nil {
|
||||
return fmt.Errorf("unknown operator '%s'", n.kind)
|
||||
}
|
||||
return
|
||||
}
|
||||
func (n operator) String() string { return n.kind }
|
||||
|
||||
func (n keyword) Validate() (err error) { return }
|
||||
func (n keyword) String() string { return n.keyword }
|
||||
|
||||
func (n interval) Validate() (err error) { return }
|
||||
func (n interval) String() string { return fmt.Sprintf("INTERVAL %s %s", n.value, n.unit) }
|
||||
|
||||
func (n function) Validate() (err error) { return }
|
||||
func (n function) String() string { return fmt.Sprintf("%s(%s)", n.name, n.arguments) }
|
||||
|
||||
func (n Ident) Validate() (err error) { return }
|
||||
func (n Ident) String() string { return n.Value }
|
||||
|
||||
func (nn parserNodes) Validate() (err error) {
|
||||
if err = validate(nn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
l := len(nn)
|
||||
if l == 0 {
|
||||
return fmt.Errorf("empty set")
|
||||
}
|
||||
|
||||
if op, ok := nn[0].(operator); ok {
|
||||
return fmt.Errorf("malformed expression, unexpected operator '%s' at first node", op)
|
||||
}
|
||||
|
||||
if l > 1 {
|
||||
if op, ok := nn[l-1].(operator); ok {
|
||||
return fmt.Errorf("malformed expression, unexpected operator '%s' at last node", op)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (nn parserNodes) String() (out string) {
|
||||
for i, n := range nn {
|
||||
if i > 0 {
|
||||
out = out + " "
|
||||
}
|
||||
|
||||
out = out + n.String()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (nn parserNodeSet) Validate() (err error) {
|
||||
return validate(nn)
|
||||
}
|
||||
|
||||
func (nn parserNodeSet) String() (out string) {
|
||||
for i, n := range nn {
|
||||
if i > 0 {
|
||||
out = out + ", "
|
||||
}
|
||||
out = out + n.String()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func validate(nn []parserNode) (err error) {
|
||||
if len(nn) == 0 {
|
||||
return fmt.Errorf("empty node set")
|
||||
}
|
||||
|
||||
for _, n := range nn {
|
||||
if err = n.Validate(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package qlng
|
||||
|
||||
type (
|
||||
tokenCode int
|
||||
)
|
||||
|
||||
const (
|
||||
CHAR_WHITELIST_WHITESPACE = " \n\t"
|
||||
CHAR_WHITELIST_OPERATORS = "!+-/*=<>&|"
|
||||
CHAR_WHITELIST_QUOTES = "'"
|
||||
)
|
||||
|
||||
const (
|
||||
// Special tokens
|
||||
ILLEGAL tokenCode = iota
|
||||
EOF
|
||||
WS // 2
|
||||
IDENT
|
||||
LNULL
|
||||
LBOOL // 4
|
||||
LNUMBER
|
||||
LSTRING
|
||||
COMMA // ,
|
||||
DOT // .
|
||||
OPERATOR // + - / *
|
||||
PARENTHESIS_OPEN
|
||||
PARENTHESIS_CLOSE
|
||||
KEYWORD
|
||||
)
|
||||
@@ -0,0 +1,159 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type (
|
||||
TokenConsumerWS struct{}
|
||||
TokenConsumerIdent struct{}
|
||||
TokenConsumerOperator struct{}
|
||||
TokenConsumerComma struct{}
|
||||
TokenConsumerString struct{}
|
||||
TokenConsumerNumber struct{}
|
||||
TokenConsumerGeneric struct {
|
||||
token tokenCode
|
||||
whitelist string
|
||||
maxLength int
|
||||
}
|
||||
)
|
||||
|
||||
func in(ch rune, wl string) bool {
|
||||
var w rune
|
||||
for _, w = range wl {
|
||||
if ch == w {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (g TokenConsumerGeneric) Test(ch rune) bool {
|
||||
return in(ch, g.whitelist)
|
||||
}
|
||||
|
||||
func (g TokenConsumerGeneric) Consume(s RuneReader) Token {
|
||||
// Create a buffer and read the current character into it.
|
||||
var buf bytes.Buffer
|
||||
buf.WriteRune(s.read())
|
||||
|
||||
// Read every subsequent whitespace character into the buffer.
|
||||
// Non-whitespace characters and EOF will cause the loop to exit.
|
||||
for {
|
||||
if g.maxLength > 0 && buf.Len() >= g.maxLength {
|
||||
// Length control
|
||||
break
|
||||
}
|
||||
|
||||
if ch := s.read(); ch == eof {
|
||||
break
|
||||
} else if !g.Test(ch) {
|
||||
s.unread()
|
||||
break
|
||||
} else {
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return Token{code: g.token, literal: buf.String()}
|
||||
}
|
||||
|
||||
func (i TokenConsumerIdent) Test(ch rune) bool {
|
||||
return isLetter(ch)
|
||||
}
|
||||
|
||||
// Consumes the current rune and all contiguous ident runes.
|
||||
func (TokenConsumerIdent) Consume(s RuneReader) Token {
|
||||
// Create a buffer and read the current character into it.
|
||||
var buf bytes.Buffer
|
||||
buf.WriteRune(s.read())
|
||||
|
||||
// Read every subsequent ident character into the buffer.
|
||||
// Non-ident characters and EOF will cause the loop to exit.
|
||||
for {
|
||||
if ch := s.read(); ch == eof {
|
||||
break
|
||||
} else if !isLetter(ch) && !isDigit(ch) && ch != '_' {
|
||||
s.unread()
|
||||
break
|
||||
} else {
|
||||
_, _ = buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
lit := strings.ToUpper(buf.String())
|
||||
|
||||
switch lit {
|
||||
case "NULL":
|
||||
return Token{code: LNULL}
|
||||
case "TRUE", "FALSE":
|
||||
return Token{code: LBOOL, literal: lit}
|
||||
case "IS", "LIKE", "NOT", "AND", "OR", "XOR":
|
||||
return Token{code: OPERATOR, literal: lit}
|
||||
case "DESC", "ASC", "INTERVAL":
|
||||
return Token{code: KEYWORD, literal: lit}
|
||||
}
|
||||
|
||||
// Otherwise return as a regular identifier.
|
||||
return Token{code: IDENT, literal: buf.String()}
|
||||
}
|
||||
|
||||
func (str TokenConsumerString) Test(ch rune) bool {
|
||||
return in(ch, CHAR_WHITELIST_QUOTES)
|
||||
}
|
||||
|
||||
// Consumes entire string (skipping quotes)
|
||||
func (str TokenConsumerString) Consume(s RuneReader) Token {
|
||||
var buf bytes.Buffer
|
||||
var escaping = false
|
||||
var ch = s.read() // skip quite
|
||||
|
||||
for {
|
||||
if ch = s.read(); ch == eof {
|
||||
break
|
||||
} else if !escaping && str.Test(ch) { // test for quote
|
||||
return Token{code: LSTRING, literal: buf.String()}
|
||||
} else {
|
||||
escaping = !escaping && ch == '\\'
|
||||
if !escaping {
|
||||
// Add char to buffer if not escaping
|
||||
_, _ = buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This string did not end properly (with an enclosing quote).
|
||||
return Token{code: ILLEGAL, literal: buf.String() + string(ch)}
|
||||
}
|
||||
|
||||
func (str TokenConsumerNumber) Test(ch rune) bool {
|
||||
return isDigit(ch)
|
||||
}
|
||||
|
||||
// Consumes entire number (very naive and simplified)
|
||||
func (str TokenConsumerNumber) Consume(s RuneReader) Token {
|
||||
// Create a buffer and read the current character into it.
|
||||
var buf bytes.Buffer
|
||||
buf.WriteRune(s.read())
|
||||
|
||||
for {
|
||||
if ch := s.read(); ch == eof {
|
||||
break
|
||||
} else if !isDigit(ch) {
|
||||
s.unread()
|
||||
break
|
||||
} else {
|
||||
_, _ = buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise return as a regular identifier.
|
||||
return Token{code: LNUMBER, literal: buf.String()}
|
||||
}
|
||||
|
||||
// isLetter returns true if the rune is a letter.
|
||||
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
|
||||
|
||||
// isDigit returns true if the rune is a digit.
|
||||
func isDigit(ch rune) bool { return ch >= '0' && ch <= '9' }
|
||||
@@ -0,0 +1,20 @@
|
||||
package qlng
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
truthy = regexp.MustCompile(`^(t(rue)?|y(es)?|1)$`)
|
||||
)
|
||||
|
||||
// Check what boolean value the given string conforms to
|
||||
func evalBool(v string) bool {
|
||||
return truthy.MatchString(strings.ToLower(v))
|
||||
}
|
||||
|
||||
// Check if the given string is a float
|
||||
func isFloaty(v string) bool {
|
||||
return strings.Contains(v, ".")
|
||||
}
|
||||
Reference in New Issue
Block a user