From 25a9be1a5444d0664528b0f417aada12cc65b294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Thu, 5 Aug 2021 09:31:04 +0200 Subject: [PATCH] Refactor pkg/ql for more flexibility Created a temporary pkg/qlng package which contains the reworked logic to avoid (potentially) breaking the rest of the system. Most of the testing will be done implicitly via reporting tests. --- pkg/qlng/ast_nodes.go | 231 +++++++++++++++++++++++++++ pkg/qlng/ast_nodes_test.go | 32 ++++ pkg/qlng/lexer.go | 112 ++++++++++++++ pkg/qlng/lexer_test.go | 105 +++++++++++++ pkg/qlng/parser.go | 301 ++++++++++++++++++++++++++++++++++++ pkg/qlng/parser_nodes.go | 204 ++++++++++++++++++++++++ pkg/qlng/token_codes.go | 29 ++++ pkg/qlng/token_consumers.go | 159 +++++++++++++++++++ pkg/qlng/util.go | 20 +++ 9 files changed, 1193 insertions(+) create mode 100644 pkg/qlng/ast_nodes.go create mode 100644 pkg/qlng/ast_nodes_test.go create mode 100644 pkg/qlng/lexer.go create mode 100644 pkg/qlng/lexer_test.go create mode 100644 pkg/qlng/parser.go create mode 100644 pkg/qlng/parser_nodes.go create mode 100644 pkg/qlng/token_codes.go create mode 100644 pkg/qlng/token_consumers.go create mode 100644 pkg/qlng/util.go diff --git a/pkg/qlng/ast_nodes.go b/pkg/qlng/ast_nodes.go new file mode 100644 index 000000000..5e16d8565 --- /dev/null +++ b/pkg/qlng/ast_nodes.go @@ -0,0 +1,231 @@ +package qlng + +type ( + ASTNode struct { + pMeta *parserMeta + + Ref string `json:"ref,omitempty"` + Args ASTNodeSet `json:"args,omitempty"` + + Symbol string `json:"symbol,omitempty"` + Value *typedValueWrap `json:"value,omitempty"` + + Raw string `json:"raw,omitempty"` + } + ASTNodeSet []*ASTNode + typedValueWrap struct { + Value interface{} `json:"@value"` + Type string `json:"@type"` + } + + parserMeta struct { + opDef *opDef + } +) + +func MakeValueOf(t string, v interface{}) *typedValueWrap { + return &typedValueWrap{ + Type: t, + Value: v, + } +} + +// Traverse traverses the AST down to leaf nodes. +// +// If fnc. returns false, the traversal of the current branch ends. +func (n *ASTNode) Traverse(f func(*ASTNode) (bool, *ASTNode, error)) (err error) { + var ok bool + var r *ASTNode + if n == nil { + return nil + } + + ok, r, err = f(n) + if err != nil { + return err + } + *n = *r + if !ok { + return + } + + for _, a := range n.Args { + if err = a.Traverse(f); err != nil { + return + } + } + + return +} + +func (n ASTNode) Clone() *ASTNode { + aa := n.Args + + if n.Args != nil { + n.Args = make(ASTNodeSet, len(aa)) + for i, a := range aa { + n.Args[i] = a.Clone() + } + } + + if n.Value != nil { + n.Value = &typedValueWrap{ + Type: n.Value.Type, + Value: n.Value.Value, + } + } + + return &n +} + +func (n lNull) ToAST() (out *ASTNode) { + return &ASTNode{ + Ref: "null", + } +} + +func (n lBoolean) ToAST() (out *ASTNode) { + return &ASTNode{ + Value: MakeValueOf("Boolean", n.value), + } +} + +func (n lString) ToAST() (out *ASTNode) { + return &ASTNode{ + Value: MakeValueOf("String", n.value), + } +} + +// @todo differentiate between floats and others +func (n lNumber) ToAST() (out *ASTNode) { + if isFloaty(n.value) { + return &ASTNode{ + Value: MakeValueOf("Float", n.value), + } + } else { + return &ASTNode{ + Value: MakeValueOf("Int", n.value), + } + } +} + +func (n operator) ToAST() (out *ASTNode) { + op := getOp(n.kind) + + return &ASTNode{ + Ref: op.name, + Args: make(ASTNodeSet, 0, 2), + pMeta: &parserMeta{opDef: op}, + } +} + +func (n Ident) ToAST() (out *ASTNode) { + return &ASTNode{ + Symbol: n.Value, + } +} + +func (n keyword) ToAST() (out *ASTNode) { + return &ASTNode{ + Ref: n.keyword, + } +} + +func (n interval) ToAST() (out *ASTNode) { + return &ASTNode{ + Ref: "interval", + Args: ASTNodeSet{ + {Symbol: n.unit}, + {Value: MakeValueOf("Number", n.value)}, + }, + } +} + +func (n function) ToAST() (out *ASTNode) { + auxA := n.arguments.ToAST() + + return &ASTNode{ + Ref: n.name, + Args: auxA.Args, + } +} + +func (nn parserNodeSet) ToAST() (out *ASTNode) { + auxArgs := make(ASTNodeSet, 0, len(nn)) + + for _, n := range nn { + auxArgs = append(auxArgs, n.ToAST()) + } + + return &ASTNode{ + Ref: "group", + Args: auxArgs, + } +} + +func (nn parserNodes) ToAST() (out *ASTNode) { + // Prep + auxArgs := make(ASTNodeSet, 0, len(nn)) + + // Convert the entire level to AST nodes + for _, n := range nn { + auxArgs = append(auxArgs, n.ToAST()) + } + + // In the current level, have the operators consume operands + // based on their defined weight + // + // - find the highest prio. op and have it consume what it wants + // - repeat until all ops are satisfied + // -- post optimizations? + for { + var bestOp *opDef + bestOpIx := -1 + + // We're done when it is reduced to 1 + if len(auxArgs) <= 1 { + break + } + + for _i, _a := range auxArgs { + i := _i + a := _a + + // use this as a delimiter for now + if a.pMeta == nil || a.pMeta.opDef == nil { + continue + } + + if bestOp == nil { + bestOp = a.pMeta.opDef + bestOpIx = i + continue + } + + if a.pMeta.opDef.weight < bestOp.weight { + bestOp = a.pMeta.opDef + bestOpIx = i + continue + } + } + + // Have the op consume what it needs. + // Currently we only have binary operators (the !unary ones) so this is fine. + arg := auxArgs[bestOpIx] + arg.Args = append(arg.Args, auxArgs[bestOpIx-1], auxArgs[bestOpIx+1]) + // this is not needed anymore so we can remove it + arg.pMeta = nil + + // Remove the consumed bits and replace it with the new bit + aux := auxArgs[0 : bestOpIx-1] + aux = append(aux, arg) + // +1 for right side, +1 because the left index is inclusive + aux = append(aux, auxArgs[bestOpIx+2:]...) + auxArgs = aux + } + + return &ASTNode{ + Ref: "group", + Args: auxArgs, + } +} diff --git a/pkg/qlng/ast_nodes_test.go b/pkg/qlng/ast_nodes_test.go new file mode 100644 index 000000000..f7f28b96b --- /dev/null +++ b/pkg/qlng/ast_nodes_test.go @@ -0,0 +1,32 @@ +package qlng + +import ( + "testing" +) + +// Ensure the parser can parse strings into Statement ASTs. +func Test_Validators(t *testing.T) { + var tests = []struct { + tree parserNode + }{ + { + tree: parserNodes{ + Ident{Value: "foo"}, + operator{kind: "="}, + }, + }, + { + tree: parserNodes{ + operator{kind: "="}, + Ident{Value: "foo"}, + }, + }, + } + + for i, test := range tests { + if err := test.tree.Validate(); err == nil { + t.Fatalf("expecting error, got nil:\n"+ + " test case: %d. %q", i, test.tree.String()) + } + } +} diff --git a/pkg/qlng/lexer.go b/pkg/qlng/lexer.go new file mode 100644 index 000000000..cc1f2372d --- /dev/null +++ b/pkg/qlng/lexer.go @@ -0,0 +1,112 @@ +package qlng + +import ( + "bufio" + "io" +) + +// Lexer represents a lexical scanner. +type ( + RuneReader interface { + read() rune + unread() + } + + TokenConsumers interface { + Test(ch rune) bool + Consume(s RuneReader) Token + } + + Token struct { + code tokenCode + literal string + line uint + char uint + } + + Lexer struct { + r *bufio.Reader + consumers []TokenConsumers + line uint + char uint + } +) + +// eof represents a marker rune for the end of the reader. +var eof = rune(0) + +// NewLexer returns a new instance of Lexer. +func NewLexer(r io.Reader) *Lexer { + return &Lexer{ + r: bufio.NewReader(r), + consumers: []TokenConsumers{ + &TokenConsumerGeneric{token: WS, whitelist: CHAR_WHITELIST_WHITESPACE}, + // @todo ensure operator order (eg != is valid, =! is not) + &TokenConsumerGeneric{token: OPERATOR, whitelist: CHAR_WHITELIST_OPERATORS}, + &TokenConsumerGeneric{token: COMMA, whitelist: ",", maxLength: 1}, + &TokenConsumerGeneric{token: DOT, whitelist: ".", maxLength: 1}, + &TokenConsumerGeneric{token: PARENTHESIS_OPEN, whitelist: "(", maxLength: 1}, + &TokenConsumerGeneric{token: PARENTHESIS_CLOSE, whitelist: ")", maxLength: 1}, + &TokenConsumerString{}, + &TokenConsumerNumber{}, + &TokenConsumerIdent{}, + }, + } +} + +// Scan returns the next token and literal value. +func (s *Lexer) Scan() Token { + var ch = s.peek() + + if ch == '\n' { + s.line++ + s.char = 0 + } + + if eof == ch { + return Token{code: EOF, line: s.line, char: s.char} + } + + for _, c := range s.consumers { + if c.Test(ch) { + t := c.Consume(s) + t.line = s.line + t.char = s.char + return t + } + } + + return Token{code: ILLEGAL, literal: string(ch), line: s.line, char: s.char} +} + +// read reads the next rune from the buffered reader. +// Returns the rune(0) if an error occurs (or io.EOF is returned). +func (s *Lexer) read() rune { + ch, _, err := s.r.ReadRune() + s.char++ + if err != nil { + return eof + } + return ch +} + +func (s *Lexer) peek() rune { + bb, err := s.r.Peek(1) + if err != nil || len(bb) == 0 { + return eof + } + return rune(bb[0]) +} + +// unread places the previously read rune back on the reader. +func (s *Lexer) unread() { _ = s.r.UnreadRune() } + +func (t Token) Is(cc ...tokenCode) bool { + for _, c := range cc { + if t.code == c { + return true + } + } + + return false +} diff --git a/pkg/qlng/lexer_test.go b/pkg/qlng/lexer_test.go new file mode 100644 index 000000000..e4aece0d6 --- /dev/null +++ b/pkg/qlng/lexer_test.go @@ -0,0 +1,105 @@ +package qlng + +import ( + "strings" + "testing" +) + +// Ensure the scanner can scan tokens correctly. +func TestScanner_ScanSimple(t *testing.T) { + var tests = []struct { + s string + tok tokenCode + lit string + }{ + // Special tokens (EOF, ILLEGAL, WS) + {s: ``, tok: EOF}, + {s: `#`, tok: ILLEGAL, lit: `#`}, + {s: ` `, tok: WS, lit: " "}, + {s: "\t", tok: WS, lit: "\t"}, + {s: "\n", tok: WS, lit: "\n"}, + + // Operators + {s: `*`, tok: OPERATOR, lit: "*"}, + {s: `!=`, tok: OPERATOR, lit: "!="}, + {s: `<`, tok: OPERATOR, lit: "<"}, + {s: `>`, tok: OPERATOR, lit: ">"}, + {s: `>=`, tok: OPERATOR, lit: ">="}, + {s: `<>`, tok: OPERATOR, lit: "<>"}, + {s: `+`, tok: OPERATOR, lit: "+"}, + {s: `'fooo'`, tok: LSTRING, lit: "fooo"}, + {s: `'escaped \' quote'`, tok: LSTRING, lit: "escaped ' quote"}, + {s: `'double \\ escape'`, tok: LSTRING, lit: "double \\ escape"}, + {s: `12345`, tok: LNUMBER, lit: "12345"}, + + // Identifiers + {s: `foo`, tok: IDENT, lit: `foo`}, + {s: `Zx12_3U_-`, tok: IDENT, lit: `Zx12_3U_`}, + + // Parenthesis + {s: `(`, tok: PARENTHESIS_OPEN, lit: `(`}, + {s: `)`, tok: PARENTHESIS_CLOSE, lit: `)`}, + + // Literals + {s: `true`, tok: LBOOL, lit: `TRUE`}, + } + + for i, test := range tests { + s := NewLexer(strings.NewReader(test.s)) + tok := s.Scan() + if test.tok != tok.code { + t.Errorf("%d. %q token mismatch: exp=%d got=%d <%q>", i, test.s, test.tok, tok.code, tok.literal) + } else if test.lit != tok.literal { + t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, test.s, test.lit, tok.literal) + } + } +} + +func TestScanner_ScanComplex(t *testing.T) { + var tests = []struct { + s string + tokens []tokenCode + }{ + // Special tokens (EOF, ILLEGAL, WS) + {`func(arg1, arg2)`, + []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}}, + {`arg1 * arg2`, + []tokenCode{IDENT, WS, OPERATOR, WS, IDENT}}, + {`date_format(created_at,'%Y')`, + []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, LSTRING, PARENTHESIS_CLOSE}}, + {`foo LIKE 'abc%'`, + []tokenCode{IDENT, WS, OPERATOR, WS, LSTRING}}, + {`foo NOT LIKE 'abc%'`, + []tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, LSTRING}}, + {`foo DESC`, + []tokenCode{IDENT, WS, KEYWORD}}, + {`year(now())-1`, + []tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, PARENTHESIS_OPEN, PARENTHESIS_CLOSE, PARENTHESIS_CLOSE, OPERATOR, LNUMBER}}, + } + + for _, test := range tests { + var tokens []Token + s := NewLexer(strings.NewReader(test.s)) + for { + tok := s.Scan() + if tok.Is(EOF) { + break + } + + tokens = append(tokens, tok) + } + + if len(tokens) != len(test.tokens) { + t.Errorf("Collected tokens do not match (%v)", tokens) + } + + for i := 0; i < len(tokens); i++ { + if tokens[i].code != test.tokens[i] { + t.Errorf("Input: %s", test.s) + t.Errorf("Expected: %v", test.tokens) + t.Errorf("Collected: %v", tokens) + break + } + } + } +} diff --git a/pkg/qlng/parser.go b/pkg/qlng/parser.go new file mode 100644 index 000000000..b3fdb9946 --- /dev/null +++ b/pkg/qlng/parser.go @@ -0,0 +1,301 @@ +package qlng + +import ( + "fmt" + "strings" +) + +type ( + // Parser represents a parser. + Parser struct { + lexer *Lexer + tokbuf []Token + + OnIdent IdentHandler + OnFunction FunctionHandler + + // parenthesis level control + level uint + } + + IdentHandler func(ident Ident) (Ident, error) + FunctionHandler func(ident function) (parserNode, error) +) + +// NewParser returns a new instance of Parser. +func NewParser() *Parser { + p := &Parser{ + OnIdent: func(ident Ident) (Ident, error) { return ident, nil }, + OnFunction: func(ident function) (parserNode, error) { return ident, nil }, + } + + return p +} + +// Removes oldest token in the buffer, adds new one and returns 2nd oldest +func (p *Parser) nextToken() Token { + var t Token + for { + t = p.lexer.Scan() + if !t.Is(WS) { + p.tokbuf = append(p.tokbuf[1:], t) + return p.tokbuf[0] + } + } +} + +func (p *Parser) peekToken(s int) Token { + return p.tokbuf[s] +} + +func (p *Parser) initLexer(s string) { + p.lexer = NewLexer(strings.NewReader(s)) + p.tokbuf = make([]Token, 3) + + for c := 1; c < cap(p.tokbuf); c++ { + // Fill the buffer + p.nextToken() + } +} + +// Parse parses the given expression and returns the generated AST +func (p *Parser) Parse(s string) (*ASTNode, error) { + p.initLexer(s) + + if set, err := p.parse(p.nextToken()); err != nil { + return nil, err + } else if len(set) == 1 { + return set[0].ToAST(), set[0].Validate() + } else { + return set.ToAST(), set.Validate() + } + +} + +// Peek ahead if there is an alias ident () +// +// @todo remove this; not used anywhere after report builder gets removed +func (p *Parser) peekIfAlias() bool { + var f, s = p.peekToken(1), p.peekToken(2) + return f.Is(IDENT) && strings.ToUpper(f.literal) == "AS" && s.Is(IDENT) +} + +func (p *Parser) parse(t Token) (list parserNodes, err error) { + goto checkToken + +next: + if p.peekToken(1).Is(COMMA) || p.peekIfAlias() { + // Peek ahead and exit on comma + return + } + if p.peekToken(1).Is(PARENTHESIS_CLOSE) { + // Peek ahead and exit on closed parenthesis + if p.level == 0 { + return nil, fmt.Errorf("closing unopened parenthesis in expression") + } + p.level-- + return + } + + t = p.nextToken() + +checkToken: + switch t.code { + case EOF: + break + case WS: + // Ignore ws... next token... + goto next + case ILLEGAL: + return nil, fmt.Errorf("found an illegal token (%+v)", t) + case IDENT: + var ident parserNode + if ident, err = p.parseIdent(t); err != nil { + return nil, err + } else { + list = append(list, ident) + goto next + } + case LNULL: + list = append(list, lNull{}) + goto next + case LBOOL: + list = append(list, lBoolean{value: evalBool(t.literal)}) + goto next + case OPERATOR: + if len(list) > 0 { + // Merge with previous operator node + if prevOp, ok := list[len(list)-1].(operator); ok { + list[len(list)-1] = operator{kind: prevOp.kind + " " + t.literal} + goto next + } + } + + list = append(list, operator{kind: t.literal}) + goto next + case KEYWORD: + if keyword, err := p.parseKeyword(t); err != nil { + return nil, err + } else { + list = append(list, keyword) + } + goto next + case LNUMBER: + list = append(list, lNumber{value: t.literal}) + goto next + case LSTRING: + list = append(list, lString{value: t.literal}) + goto next + case PARENTHESIS_OPEN: + depth := p.level + p.level++ + if sub, err := p.parse(p.nextToken()); err != nil { + return nil, err + } else { + list = append(list, sub) + } + + // Allow parent level to continue parsing. + // Example: ((A) AND (B)) + // +1 since PARENTHESIS_CLOSE decrease level + if (p.level + 1) != depth { + p.nextToken() + goto next + } + default: + return nil, fmt.Errorf("unexpected token while parsing expression (%v)", t) + } + + return list, nil +} + +func (p *Parser) parseIdent(t Token) (list parserNode, err error) { + if p.peekToken(1).Is(PARENTHESIS_OPEN) { + // Handle function calls: ... + f := function{name: t.literal} + if f.arguments, err = p.parseSet(); err != nil { + return nil, err + } else { + return p.OnFunction(f) + } + } + + i := Ident{Value: t.literal} + + if p.peekToken(1).Is(DOT) { + p.nextToken() + i.Value += "." + l2 := p.nextToken() + if l2.Is(IDENT) { + i.Value += l2.literal + } + } + + return p.OnIdent(i) +} + +func (p *Parser) parseSet() (list parserNodeSet, err error) { + var expr parserNodes + var parenthesisOpened = false + +next: + t := p.nextToken() + + if p.peekIfAlias() { + return + } + + switch t.code { + case WS: + goto next + case PARENTHESIS_OPEN: + p.level++ + parenthesisOpened = true + goto next + case LNUMBER: + list = append(list, lNumber{value: t.literal}) + goto next + case LSTRING: + list = append(list, lString{value: t.literal}) + goto next + case OPERATOR: + list = append(list, operator{kind: t.literal}) + goto next + case KEYWORD: + if keyword, err := p.parseKeyword(t); err != nil { + return nil, err + } else { + list = append(list, keyword) + } + goto next + case IDENT: + if p.peekToken(1).Is(OPERATOR) { + // Looks like we have an expression ahead of us + + if parenthesisOpened { + // Expression will find closing parenthesis and dec. the level + // so, lets bump up the number + p.level++ + } + + if expr, err = p.parse(t); err != nil { + return + } + + list = append(list, expr) + goto next + } + + var ident parserNode + if ident, err = p.parseIdent(t); err != nil { + return nil, err + } else { + list = append(list, ident) + + goto next + } + case COMMA: + goto next + case EOF: + return + case PARENTHESIS_CLOSE: + // Peek ahead and exit on closed parenthesis + if p.level == 0 { + return nil, fmt.Errorf("closing unopened parenthesis in set") + } + p.level-- + return + default: + return nil, fmt.Errorf("unexpected token while parsing set (%v)", t) + } +} + +func (p *Parser) parseKeyword(t Token) (list parserNode, err error) { + switch strings.ToUpper(t.literal) { + case "INTERVAL": + i := interval{value: p.nextToken().literal} + u := p.nextToken() + + if u.code != IDENT { + return nil, fmt.Errorf("expecting identifier, got %v", t) + } else { + switch strings.ToUpper(u.literal) { + case "MICROSECOND", "SECOND", "MINUTE", "HOUR", + "DAY", "WEEK", "MONTH", "QUARTER", "YEAR", + "SECOND_MICROSECOND", "MINUTE_MICROSECOND", "MINUTE_SECOND", "HOUR_MICROSECOND", "HOUR_SECOND", + "HOUR_MINUTE", "DAY_MICROSECOND", "DAY_SECOND", "DAY_MINUTE", "DAY_HOUR", "YEAR_MONTH": + // All good + break + default: + return nil, fmt.Errorf("expecting interval unit, got %v", u.literal) + } + } + + i.unit = u.literal + + return i, nil + + default: + return keyword{keyword: t.literal}, nil + } +} diff --git a/pkg/qlng/parser_nodes.go b/pkg/qlng/parser_nodes.go new file mode 100644 index 000000000..1eb95c5fc --- /dev/null +++ b/pkg/qlng/parser_nodes.go @@ -0,0 +1,204 @@ +package qlng + +import ( + "fmt" + "strings" +) + +// SelectStatement represents a SQL SELECT statement. +type ( + parserNode interface { + fmt.Stringer + + Validate() error + ToAST() *ASTNode + } + + parserNodeSet []parserNode // Stream of comma delimited nodes + parserNodes []parserNode // Stream of space delimited nodes + + lNull struct{} + lBoolean struct{ value bool } + + lString struct { + value string + args []interface{} + } + + lNumber struct { + value string + } + + operator struct { + kind string + } + + Ident struct { + Value string + args []interface{} + } + + keyword struct { + keyword string + } + + interval struct { + value string + unit string + } + + function struct { + name string + arguments parserNodeSet + } + + opDef struct { + name string + weight int + } +) + +var ( + ops = map[string]opDef{ + // generic comparison + `=`: {name: `eq`, weight: 40}, + `==`: {name: `eq`, weight: 40}, + `===`: {name: `eq`, weight: 40}, + `!=`: {name: `ne`, weight: 40}, + `!==`: {name: `ne`, weight: 40}, + `<>`: {name: `ne`, weight: 40}, + `<`: {name: `lt`, weight: 30}, + `<=`: {name: `le`, weight: 30}, + `>`: {name: `gt`, weight: 30}, + `>=`: {name: `ge`, weight: 30}, + `IS`: {name: `is`, weight: 40}, + `IS NOT`: {name: `nis`, weight: 40}, + + // conjunction + `AND`: {name: `and`, weight: 50}, + `&&`: {name: `and`, weight: 50}, + `OR`: {name: `or`, weight: 60}, + `||`: {name: `or`, weight: 60}, + `XOR`: {name: `xor`, weight: 60}, + + // math + `+`: {name: `add`, weight: 20}, + `-`: {name: `sub`, weight: 20}, + `*`: {name: `mult`, weight: 10}, + `/`: {name: `div`, weight: 10}, + + // str comp. + `LIKE`: {name: `like`, weight: 40}, + `NOT LIKE`: {name: `nlike`, weight: 40}, + } +) + +func getOp(op string) *opDef { + o, ok := ops[strings.ToUpper(op)] + if !ok { + return nil + } + return &o +} + +func (n lNull) Validate() (err error) { return } +func (n lNull) String() string { return "NULL" } + +func (n lBoolean) Validate() (err error) { return } +func (n lBoolean) String() string { + if n.value { + return "TRUE" + } else { + return "FALSE" + } +} + +func (n lString) Validate() (err error) { return } +func (n lString) String() string { return fmt.Sprintf("%q", n.value) } + +func (n lNumber) Validate() (err error) { return } +func (n lNumber) String() string { return n.value } + +func (n operator) Validate() (err error) { + if getOp(n.kind) == nil { + return fmt.Errorf("unknown operator '%s'", n.kind) + } + return +} +func (n operator) String() string { return n.kind } + +func (n keyword) Validate() (err error) { return } +func (n keyword) String() string { return n.keyword } + +func (n interval) Validate() (err error) { return } +func (n interval) String() string { return fmt.Sprintf("INTERVAL %s %s", n.value, n.unit) } + +func (n function) Validate() (err error) { return } +func (n function) String() string { return fmt.Sprintf("%s(%s)", n.name, n.arguments) } + +func (n Ident) Validate() (err error) { return } +func (n Ident) String() string { return n.Value } + +func (nn parserNodes) Validate() (err error) { + if err = validate(nn); err != nil { + return + } + + l := len(nn) + if l == 0 { + return fmt.Errorf("empty set") + } + + if op, ok := nn[0].(operator); ok { + return fmt.Errorf("malformed expression, unexpected operator '%s' at first node", op) + } + + if l > 1 { + if op, ok := nn[l-1].(operator); ok { + return fmt.Errorf("malformed expression, unexpected operator '%s' at last node", op) + } + } + + return +} + +func (nn parserNodes) String() (out string) { + for i, n := range nn { + if i > 0 { + out = out + " " + } + + out = out + n.String() + } + + return +} + +func (nn parserNodeSet) Validate() (err error) { + return validate(nn) +} + +func (nn parserNodeSet) String() (out string) { + for i, n := range nn { + if i > 0 { + out = out + ", " + } + out = out + n.String() + } + + return +} + +func validate(nn []parserNode) (err error) { + if len(nn) == 0 { + return fmt.Errorf("empty node set") + } + + for _, n := range nn { + if err = n.Validate(); err != nil { + return + } + } + + return +} diff --git a/pkg/qlng/token_codes.go b/pkg/qlng/token_codes.go new file mode 100644 index 000000000..2d63b5a33 --- /dev/null +++ b/pkg/qlng/token_codes.go @@ -0,0 +1,29 @@ +package qlng + +type ( + tokenCode int +) + +const ( + CHAR_WHITELIST_WHITESPACE = " \n\t" + CHAR_WHITELIST_OPERATORS = "!+-/*=<>&|" + CHAR_WHITELIST_QUOTES = "'" +) + +const ( + // Special tokens + ILLEGAL tokenCode = iota + EOF + WS // 2 + IDENT + LNULL + LBOOL // 4 + LNUMBER + LSTRING + COMMA // , + DOT // . + OPERATOR // + - / * + PARENTHESIS_OPEN + PARENTHESIS_CLOSE + KEYWORD +) diff --git a/pkg/qlng/token_consumers.go b/pkg/qlng/token_consumers.go new file mode 100644 index 000000000..c41c86914 --- /dev/null +++ b/pkg/qlng/token_consumers.go @@ -0,0 +1,159 @@ +package qlng + +import ( + "bytes" + "strings" +) + +type ( + TokenConsumerWS struct{} + TokenConsumerIdent struct{} + TokenConsumerOperator struct{} + TokenConsumerComma struct{} + TokenConsumerString struct{} + TokenConsumerNumber struct{} + TokenConsumerGeneric struct { + token tokenCode + whitelist string + maxLength int + } +) + +func in(ch rune, wl string) bool { + var w rune + for _, w = range wl { + if ch == w { + return true + } + } + return false +} + +func (g TokenConsumerGeneric) Test(ch rune) bool { + return in(ch, g.whitelist) +} + +func (g TokenConsumerGeneric) Consume(s RuneReader) Token { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + // Read every subsequent whitespace character into the buffer. + // Non-whitespace characters and EOF will cause the loop to exit. + for { + if g.maxLength > 0 && buf.Len() >= g.maxLength { + // Length control + break + } + + if ch := s.read(); ch == eof { + break + } else if !g.Test(ch) { + s.unread() + break + } else { + buf.WriteRune(ch) + } + } + + return Token{code: g.token, literal: buf.String()} +} + +func (i TokenConsumerIdent) Test(ch rune) bool { + return isLetter(ch) +} + +// Consumes the current rune and all contiguous ident runes. +func (TokenConsumerIdent) Consume(s RuneReader) Token { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + // Read every subsequent ident character into the buffer. + // Non-ident characters and EOF will cause the loop to exit. + for { + if ch := s.read(); ch == eof { + break + } else if !isLetter(ch) && !isDigit(ch) && ch != '_' { + s.unread() + break + } else { + _, _ = buf.WriteRune(ch) + } + } + + lit := strings.ToUpper(buf.String()) + + switch lit { + case "NULL": + return Token{code: LNULL} + case "TRUE", "FALSE": + return Token{code: LBOOL, literal: lit} + case "IS", "LIKE", "NOT", "AND", "OR", "XOR": + return Token{code: OPERATOR, literal: lit} + case "DESC", "ASC", "INTERVAL": + return Token{code: KEYWORD, literal: lit} + } + + // Otherwise return as a regular identifier. + return Token{code: IDENT, literal: buf.String()} +} + +func (str TokenConsumerString) Test(ch rune) bool { + return in(ch, CHAR_WHITELIST_QUOTES) +} + +// Consumes entire string (skipping quotes) +func (str TokenConsumerString) Consume(s RuneReader) Token { + var buf bytes.Buffer + var escaping = false + var ch = s.read() // skip quite + + for { + if ch = s.read(); ch == eof { + break + } else if !escaping && str.Test(ch) { // test for quote + return Token{code: LSTRING, literal: buf.String()} + } else { + escaping = !escaping && ch == '\\' + if !escaping { + // Add char to buffer if not escaping + _, _ = buf.WriteRune(ch) + } + } + } + + // This string did not end properly (with an enclosing quote). + return Token{code: ILLEGAL, literal: buf.String() + string(ch)} +} + +func (str TokenConsumerNumber) Test(ch rune) bool { + return isDigit(ch) +} + +// Consumes entire number (very naive and simplified) +func (str TokenConsumerNumber) Consume(s RuneReader) Token { + // Create a buffer and read the current character into it. + var buf bytes.Buffer + buf.WriteRune(s.read()) + + for { + if ch := s.read(); ch == eof { + break + } else if !isDigit(ch) { + s.unread() + break + } else { + _, _ = buf.WriteRune(ch) + } + } + + // Otherwise return as a regular identifier. + return Token{code: LNUMBER, literal: buf.String()} +} + +// isLetter returns true if the rune is a letter. +func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } + +// isDigit returns true if the rune is a digit. +func isDigit(ch rune) bool { return ch >= '0' && ch <= '9' } diff --git a/pkg/qlng/util.go b/pkg/qlng/util.go new file mode 100644 index 000000000..d707d8fb4 --- /dev/null +++ b/pkg/qlng/util.go @@ -0,0 +1,20 @@ +package qlng + +import ( + "regexp" + "strings" +) + +var ( + truthy = regexp.MustCompile(`^(t(rue)?|y(es)?|1)$`) +) + +// Check what boolean value the given string conforms to +func evalBool(v string) bool { + return truthy.MatchString(strings.ToLower(v)) +} + +// Check if the given string is a float +func isFloaty(v string) bool { + return strings.Contains(v, ".") +}