From 452fd1ca9d79567793d4546a301f1bfaca28093f Mon Sep 17 00:00:00 2001 From: Vivek Patel Date: Thu, 21 Jul 2022 16:22:30 +0530 Subject: [PATCH] Add QL Functions for RDBMS drives It will translate dateTime functions inside query as per the driver language. --- store/adapters/rdbms/dal/model.go | 1 + store/adapters/rdbms/drivers/dialect.go | 4 + store/adapters/rdbms/drivers/mysql/dialect.go | 5 + .../rdbms/drivers/postgres/dialect.go | 25 +- store/adapters/rdbms/drivers/postgres/ql.go | 155 ++++++++++-- .../rdbms/drivers/postgres/ql_test.go | 81 ++++++ .../adapters/rdbms/drivers/sqlite/dialect.go | 5 + store/adapters/rdbms/drivers/sqlite/ql.go | 237 +++++++++-------- .../adapters/rdbms/drivers/sqlite/ql_test.go | 86 +++++++ store/adapters/rdbms/ql/converter.go | 10 +- store/adapters/rdbms/ql/converter_test.go | 25 ++ store/adapters/rdbms/ql/ref.go | 87 ++++--- tests/dal/{dal_utils.go => dal_utils_test.go} | 0 tests/dal/helper.go | 238 ------------------ tests/dal/main_test.go | 232 ++++++++++++++++- 15 files changed, 777 insertions(+), 414 deletions(-) create mode 100644 store/adapters/rdbms/drivers/postgres/ql_test.go create mode 100644 store/adapters/rdbms/drivers/sqlite/ql_test.go rename tests/dal/{dal_utils.go => dal_utils_test.go} (100%) delete mode 100644 tests/dal/helper.go diff --git a/store/adapters/rdbms/dal/model.go b/store/adapters/rdbms/dal/model.go index 542f5375d..ea2f673c5 100644 --- a/store/adapters/rdbms/dal/model.go +++ b/store/adapters/rdbms/dal/model.go @@ -64,6 +64,7 @@ func Model(m *dal.Model, c queryRunner, d drivers.Dialect) *model { return ms.table.AttributeExpression(node.Symbol) }), + ql.RefHandler(d.ExprHandler), ) return ms diff --git a/store/adapters/rdbms/drivers/dialect.go b/store/adapters/rdbms/drivers/dialect.go index eb73a78db..5367d9acf 100644 --- a/store/adapters/rdbms/drivers/dialect.go +++ b/store/adapters/rdbms/drivers/dialect.go @@ -2,6 +2,7 @@ package drivers import ( "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/ql" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -34,5 +35,8 @@ type ( // NativeColumnType converts column type to type that can be used in the underlying rdbms NativeColumnType(columnType ddl.ColumnType) string + + // ExprHandler returns driver specific expression handling + ExprHandler(*ql.ASTNode, ...exp.Expression) (exp.Expression, error) } ) diff --git a/store/adapters/rdbms/drivers/mysql/dialect.go b/store/adapters/rdbms/drivers/mysql/dialect.go index a050bf5fa..42e3b7094 100644 --- a/store/adapters/rdbms/drivers/mysql/dialect.go +++ b/store/adapters/rdbms/drivers/mysql/dialect.go @@ -3,6 +3,7 @@ package mysql import ( "fmt" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" "strconv" "strings" @@ -102,6 +103,10 @@ func (mysqlDialect) NativeColumnType(ct ddl.ColumnType) string { return columnTypeTranslator(ct) } +func (mysqlDialect) ExprHandler(n *ql.ASTNode, args ...exp.Expression) (exp.Expression, error) { + return ql.DefaultRefHandler(n, args...) +} + func JSONPath(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { var ( sql strings.Builder diff --git a/store/adapters/rdbms/drivers/postgres/dialect.go b/store/adapters/rdbms/drivers/postgres/dialect.go index 2e24ca9a7..47fa3fc60 100644 --- a/store/adapters/rdbms/drivers/postgres/dialect.go +++ b/store/adapters/rdbms/drivers/postgres/dialect.go @@ -4,38 +4,39 @@ import ( "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" ) type ( - mysqlDialect struct{} + postgresDialect struct{} ) var ( - _ drivers.Dialect = &mysqlDialect{} + _ drivers.Dialect = &postgresDialect{} - dialect = &mysqlDialect{} + dialect = &postgresDialect{} goquDialectWrapper = goqu.Dialect("postgres") ) -func Dialect() *mysqlDialect { +func Dialect() *postgresDialect { return dialect } -func (mysqlDialect) GOQU() goqu.DialectWrapper { +func (postgresDialect) GOQU() goqu.DialectWrapper { return goquDialectWrapper } -func (mysqlDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { +func (postgresDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { return drivers.DeepIdentJSON(ident, pp...), nil } -func (d mysqlDialect) TableCodec(m *dal.Model) drivers.TableCodec { +func (d postgresDialect) TableCodec(m *dal.Model) drivers.TableCodec { return drivers.NewTableCodec(m, d) } -func (d mysqlDialect) TypeWrap(dt dal.Type) drivers.Type { +func (d postgresDialect) TypeWrap(dt dal.Type) drivers.Type { // Any exception to general type-wrap implementation in the drivers package // should be placed here switch c := dt.(type) { @@ -46,7 +47,7 @@ func (d mysqlDialect) TypeWrap(dt dal.Type) drivers.Type { return drivers.TypeWrap(dt) } -func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func (postgresDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { var ( c exp.CastExpression ) @@ -72,6 +73,10 @@ func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression return exp.NewLiteralExpression("?", c), nil } -func (mysqlDialect) NativeColumnType(ct ddl.ColumnType) string { +func (postgresDialect) NativeColumnType(ct ddl.ColumnType) string { return ddl.ColumnTypeTranslator(ct) } + +func (postgresDialect) ExprHandler(n *ql.ASTNode, args ...exp.Expression) (exp.Expression, error) { + return ref2exp.RefHandler(n, args...) +} diff --git a/store/adapters/rdbms/drivers/postgres/ql.go b/store/adapters/rdbms/drivers/postgres/ql.go index b8e2a1c36..0fe4297e1 100644 --- a/store/adapters/rdbms/drivers/postgres/ql.go +++ b/store/adapters/rdbms/drivers/postgres/ql.go @@ -1,29 +1,130 @@ package postgres -//var ( -// sqlExprRegistry = map[string]rdbms.HandlerSig{ -// // functions -// // - filtering -// "quarter": makeGenericExtrFncHandler("QUARTER"), -// "year": makeGenericExtrFncHandler("YEAR"), -// "month": makeGenericExtrFncHandler("MONTH"), -// "date": makeGenericExtrFncHandler("DAY"), -// } -//) -// -//func sqlASTFormatter(n *ql.ASTNode) rdbms.HandlerSig { -// return sqlExprRegistry[n.Ref] -//} -// -//func makeGenericExtrFncHandler(extr string) rdbms.HandlerSig { -// return func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("EXTRACT(%s FROM %s)", extr, aa[0].S) -// args = aa[0].Args -// return -// } -//} +import ( + "fmt" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" + "github.com/doug-martin/goqu/v9/exp" + "strings" +) + +var ( + ref2exp = ql.ExprHandlerMap{ + // filtering + "now": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("NOW") + }, + }, + "quarter": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("EXTRACT", + exp.NewLiteralExpression("QUARTER FROM TIMESTAMP ?", args[0]), + ) + }, + }, + "year": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("EXTRACT", + exp.NewLiteralExpression("YEAR FROM TIMESTAMP ?", args[0]), + ) + }, + }, + "month": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("EXTRACT", + exp.NewLiteralExpression("MONTH FROM TIMESTAMP ?", args[0]), + ) + }, + }, + "timestamp": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("?::TIMESTAMPTZ", args[0]) + }, + }, + "date": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("?::DATE", args[0]) + }, + }, + "time": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("DATE_TRUNC('second', ?::TIME)::TIME", args[0]) + }, + }, + + // @todo replace given argument before constructing sql + "date_format": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("TO_CHAR", + exp.NewLiteralExpression("?::TIMESTAMPTZ", args[0]), + exp.NewLiteralExpression("?::TEXT", translateDateFormatParam(args[1])), + ) + }, + }, + + // functions currently unsupported in PostgreSQL store backend + //"DATE_ADD": { + // Handler: func(args ...exp.Expression) exp.Expression { + // return exp.NewLiteralExpression("") + // }, + //}, + //"DATE_SUB": { + // Handler: func(args ...exp.Expression) exp.Expression { + // return exp.NewLiteralExpression("") + // }, + //}, + //"STD": { + // Handler: func(args ...exp.Expression) exp.Expression { + // return exp.NewLiteralExpression("") + // }, + //}, + }.ExprHandlers() +) + +func translateDateFormatParam(e interface{}) interface{} { + le, ok := e.(exp.LiteralExpression) + if !ok { + return e + } + + args := le.Args() + if len(args) > 0 { + return dateFormatReplacer(fmt.Sprintf("%s", args[0])) + } + + return e +} + +func dateFormatReplacer(s string) string { + return strings.NewReplacer( + // @todo Doing ...%dT%H... (for iso timestamp) pgsql doesn't format it correctly + // so I'm covering this edge case. + // We should fix this properly when we redo record storage. + `%dT%H`, `DD"T"HH24`, + + `%a`, `Dy`, + `%b`, `Mon`, + `%c`, `FMMM`, + `%d`, `DD`, + `%e`, `FMDD`, + `%f`, `US`, + `%H`, `HH24`, + `%h`, `HH12`, + `%I`, `HH12`, + `%i`, `MI`, + `%j`, `DDD`, + `%k`, `FMHH24`, + `%l`, `FMHH12`, + `%M`, `FMMonth`, + `%m`, `MM`, + `%p`, `AM`, + `%r`, `HH12:MI:SS AM`, + `%S`, `SS`, + `%s`, `SS`, + `%T`, `HH24:MI:SS`, + `%W`, `FMDay`, + `%Y`, `YYYY`, + `%y`, `YY`, + `%%`, `%`, + ).Replace(s) +} diff --git a/store/adapters/rdbms/drivers/postgres/ql_test.go b/store/adapters/rdbms/drivers/postgres/ql_test.go new file mode 100644 index 000000000..6f733f237 --- /dev/null +++ b/store/adapters/rdbms/drivers/postgres/ql_test.go @@ -0,0 +1,81 @@ +package postgres + +import ( + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// @todo Ql functions should be under store/tests so it can be tested across all drivers along with generated tests. +// for now, Its test coverage is limited per driver. +func TestConverter(t *testing.T) { + const SELECT = "SELECT " + var ( + conv = ql.Converter(ql.RefHandler(dialect.ExprHandler)) + + cases = []struct { + qry string + sql string + args []any + }{ + { + qry: `quarter('2022-07-21')`, + sql: `EXTRACT(QUARTER FROM TIMESTAMP $1)`, + args: []any{"2022-07-21"}, + }, + { + qry: `year('2022-07-21')`, + sql: `EXTRACT(YEAR FROM TIMESTAMP $1)`, + args: []any{"2022-07-21"}, + }, + { + qry: `month('2022-07-21')`, + sql: `EXTRACT(MONTH FROM TIMESTAMP $1)`, + args: []any{"2022-07-21"}, + }, + { + qry: `timestamp('2022-07-21')`, + sql: `$1::TIMESTAMPTZ`, + args: []any{"2022-07-21"}, + }, + { + qry: `date('2022-07-21')`, + sql: `$1::DATE`, + args: []any{"2022-07-21"}, + }, + { + qry: `time('2022-07-21 12:41')`, + sql: `DATE_TRUNC('second', $1::TIME)::TIME`, + args: []any{"2022-07-21 12:41"}, + }, + { + qry: `date_format('2022-07-21','%a')`, + sql: `TO_CHAR($1::TIMESTAMPTZ, $2::TEXT)`, + args: []any{"2022-07-21", "Dy"}, + }, + } + ) + + for _, c := range cases { + t.Run(c.qry, func(t *testing.T) { + req := require.New(t) + + ee, err := conv.Parse(c.qry) + req.NoError(err) + + sql, args, err := dialect.GOQU().Select(ee).ToSQL() + req.NoError(err) + + p := strings.Index(sql, SELECT) + req.Zero(p) + + sql = sql[p+len(SELECT):] + + req.Equal(c.sql, sql) + req.Equal(c.args, args) + }) + } + +} diff --git a/store/adapters/rdbms/drivers/sqlite/dialect.go b/store/adapters/rdbms/drivers/sqlite/dialect.go index c0f353444..d217a27e3 100644 --- a/store/adapters/rdbms/drivers/sqlite/dialect.go +++ b/store/adapters/rdbms/drivers/sqlite/dialect.go @@ -4,6 +4,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/dialect/sqlite3" "github.com/doug-martin/goqu/v9/exp" @@ -59,3 +60,7 @@ func (sqliteDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpressio func (sqliteDialect) NativeColumnType(ct ddl.ColumnType) string { return columnTypeTranslator(ct) } + +func (sqliteDialect) ExprHandler(n *ql.ASTNode, args ...exp.Expression) (exp.Expression, error) { + return ref2exp.RefHandler(n, args...) +} diff --git a/store/adapters/rdbms/drivers/sqlite/ql.go b/store/adapters/rdbms/drivers/sqlite/ql.go index ec0775409..35a21bf65 100644 --- a/store/adapters/rdbms/drivers/sqlite/ql.go +++ b/store/adapters/rdbms/drivers/sqlite/ql.go @@ -1,101 +1,140 @@ package sqlite -//var ( -//sqlExprRegistry = map[string]rdbms.HandlerSig{ -// // functions -// // - filtering -// "now": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// if len(aa) != 0 { -// err = fmt.Errorf("expecting 0 arguments, got %d", len(aa)) -// return -// } -// -// out = "DATE('now')" -// return -// }, -// "quarter": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("(CAST(STRFTIME('%%m', %s) AS INTEGER) + 2) / 3", aa[0].S) -// args = aa[0].Args -// return -// }, -// "year": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("STRFTIME('%%Y', %s)", aa[0].S) -// args = aa[0].Args -// return -// }, -// "month": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("STRFTIME('%%m', %s)", aa[0].S) -// args = aa[0].Args -// return -// }, -// "date": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("STRFTIME('%%d', %s)", aa[0].S) -// args = aa[0].Args -// return -// }, -// -// // - strings -// "concat": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// selfEnclosed = true -// -// params := make([]string, len(aa)) -// for i, a := range aa { -// params[i] = a.S -// args = append(args, a.Args...) -// } -// -// out = fmt.Sprintf("(%s)", strings.Join(params, "||")) -// return -// }, -// -// // - typecast -// "float": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// selfEnclosed = true -// -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 argument, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("CAST(%s AS FLOAT)", aa[0].S) -// args = aa[0].Args -// return -// }, -// "string": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { -// selfEnclosed = true -// -// if len(aa) != 1 { -// err = fmt.Errorf("expecting 1 argument, got %d", len(aa)) -// return -// } -// -// out = fmt.Sprintf("CAST(%s AS TEXT)", aa[0].S) -// args = aa[0].Args -// return -// }, -//} -//) -// -//func sqlASTFormatter(n *ql.ASTNode) rdbms.HandlerSig { -// return sqlExprRegistry[n.Ref] -//} +import ( + "fmt" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" + "github.com/doug-martin/goqu/v9/exp" + "regexp" + "strings" +) + +var ( + ref2exp = ql.ExprHandlerMap{ + // filtering + "now": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("DATE", + exp.NewLiteralExpression("'NOW'"), + ) + }, + }, + "quarter": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("(CAST(STRFTIME('%m', ?) AS INTEGER) + 2) / 3", args[0]) + }, + }, + "year": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("STRFTIME", + exp.NewLiteralExpression("'%Y'"), + args[0], + ) + }, + }, + "month": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("STRFTIME", + exp.NewLiteralExpression("'%m'"), + args[0], + ) + }, + }, + "date": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("STRFTIME", + exp.NewLiteralExpression("'%Y-%m-%dT00:00:00Z'"), + args[0], + ) + }, + }, + "datetime": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("DATETIME", args[0]) + }, + }, + "timestamp": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("DATETIME", args[0]) + }, + }, + "date_format": { + HandlerE: func(args ...exp.Expression) (exp.Expression, error) { + format, err := supportedDateFormatParams(args[1]) + if err != nil { + return nil, err + } + + return exp.NewSQLFunctionExpression("STRFTIME", + format, + args[0], + ), nil + }, + }, + + // functions currently unsupported in SQLite store backend + //"DATE_ADD": { + // Handler: func(args ...exp.Expression) exp.Expression { + // return exp.NewLiteralExpression("") + // }, + //}, + //"DATE_SUB": { + // Handler: func(args ...exp.Expression) exp.Expression { + // return exp.NewLiteralExpression("") + // }, + //}, + //"STD": { + // Handler: func(args ...exp.Expression) exp.Expression { + // return exp.NewLiteralExpression("") + // }, + //}, + }.ExprHandlers() + + supportedSubstitutions = map[string]bool{ + "d": true, + "H": true, + "j": true, + "m": true, + "M": true, + "S": true, + "w": true, + "W": true, + "Y": true, + "%": true, + } +) + +func supportedDateFormatParams(e interface{}) (interface{}, error) { + le, ok := e.(exp.LiteralExpression) + if !ok { + return e, fmt.Errorf("unknown date format") + } + + var format string + args := le.Args() + if len(args) > 0 { + format = dateFormatReplacer(fmt.Sprintf("%s", args[0])) + } else { + return e, fmt.Errorf("date format not found") + } + + r := regexp.MustCompile(`%(?P.)`) + + for _, m := range r.FindAllStringSubmatch(format, -1) { + if len(m) == 0 { + continue + } + + if _, ok := supportedSubstitutions[m[1]]; !ok { + return e, fmt.Errorf("format substitution not supported: %%%s", m[1]) + } + } + + return format, nil +} + +func dateFormatReplacer(format string) string { + return strings.NewReplacer( + `%i`, `%M`, + `%U`, `%W`, + ).Replace(format) +} diff --git a/store/adapters/rdbms/drivers/sqlite/ql_test.go b/store/adapters/rdbms/drivers/sqlite/ql_test.go new file mode 100644 index 000000000..6f3ccd37f --- /dev/null +++ b/store/adapters/rdbms/drivers/sqlite/ql_test.go @@ -0,0 +1,86 @@ +package sqlite + +import ( + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// @todo Ql functions should be under store/tests so it can be tested across all drivers along with generated tests. +// for now, Its test coverage is limited per driver. +func TestConverter(t *testing.T) { + const SELECT = "SELECT " + var ( + conv = ql.Converter(ql.RefHandler(dialect.ExprHandler)) + + cases = []struct { + qry string + sql string + args []any + }{ + { + qry: `now()`, + sql: `DATE('NOW')`, + args: []any{}, + }, + { + qry: `quarter('2022-07-21')`, + sql: `(CAST(STRFTIME('%m', ?) AS INTEGER) + 2) / 3`, + args: []any{"2022-07-21"}, + }, + { + qry: `year('2022-07-21')`, + sql: `STRFTIME('%Y', ?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `month('2022-07-21')`, + sql: `STRFTIME('%m', ?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `date('2022-07-21')`, + sql: `STRFTIME('%Y-%m-%dT00:00:00Z', ?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `datetime('2022-07-21')`, + sql: `DATETIME(?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `timestamp('2022-07-21')`, + sql: `DATETIME(?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `date_format('2022-07-21','%d')`, + sql: `STRFTIME(?, ?)`, + args: []any{"%d", "2022-07-21"}, + }, + } + ) + + for _, c := range cases { + t.Run(c.qry, func(t *testing.T) { + req := require.New(t) + + ee, err := conv.Parse(c.qry) + req.NoError(err) + + sql, args, err := dialect.GOQU().Select(ee).ToSQL() + req.NoError(err) + + p := strings.Index(sql, SELECT) + req.Zero(p) + + sql = sql[p+len(SELECT):] + + req.Equal(c.sql, sql) + req.Equal(c.args, args) + }) + } + +} diff --git a/store/adapters/rdbms/ql/converter.go b/store/adapters/rdbms/ql/converter.go index e4948c926..a19385b27 100644 --- a/store/adapters/rdbms/ql/converter.go +++ b/store/adapters/rdbms/ql/converter.go @@ -1,8 +1,6 @@ package ql import ( - "fmt" - "github.com/cortezaproject/corteza-server/pkg/ql" "github.com/doug-martin/goqu/v9/exp" ) @@ -24,7 +22,7 @@ type ( op func(*converter) ) -// Initializes new converter +// Converter initializes new converter func Converter(oo ...op) *converter { c := &converter{ parser: ql.NewParser(), @@ -93,11 +91,7 @@ func (c *converter) Convert(n *ql.ASTNode) (_ exp.Expression, err error) { // DefaultRefHandler converts ref from the AST node using ref2exp func DefaultRefHandler(n *ql.ASTNode, args ...exp.Expression) (exp.Expression, error) { - if ref2exp[n.Ref] == nil { - return nil, fmt.Errorf("unknown ref %q", n.Ref) - } - - return ref2exp[n.Ref].Handler(args...), nil + return ref2exp.RefHandler(n, args...) } // DefaultSymbolHandler parses symbol from the AST node into an identifier diff --git a/store/adapters/rdbms/ql/converter_test.go b/store/adapters/rdbms/ql/converter_test.go index dec8ecd96..960e476de 100644 --- a/store/adapters/rdbms/ql/converter_test.go +++ b/store/adapters/rdbms/ql/converter_test.go @@ -49,6 +49,31 @@ func TestConverter(t *testing.T) { sql: `("one" + ? / ? * ? < ?)`, args: []any{int64(2), int64(3), int64(4), int64(10)}, }, + { + qry: `concat('foo', 'bar')`, + sql: `CONCAT(?, ?)`, + args: []any{"foo", "bar"}, + }, + { + qry: `quarter('2022-07-21')`, + sql: `QUARTER(?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `year('2022-07-21')`, + sql: `YEAR(?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `month('2022-07-21')`, + sql: `MONTH(?)`, + args: []any{"2022-07-21"}, + }, + { + qry: `date('2022-07-21')`, + sql: `DAY(?)`, + args: []any{"2022-07-21"}, + }, } ) diff --git a/store/adapters/rdbms/ql/ref.go b/store/adapters/rdbms/ql/ref.go index e68d5fac5..b9d0f2b80 100644 --- a/store/adapters/rdbms/ql/ref.go +++ b/store/adapters/rdbms/ql/ref.go @@ -1,17 +1,22 @@ package ql import ( + "fmt" + "github.com/cortezaproject/corteza-server/pkg/ql" "github.com/doug-martin/goqu/v9/exp" ) type ( - exprHandler struct { - Handler func(...exp.Expression) exp.Expression + ExprHandlerMap map[string]*ExprHandler + + ExprHandler struct { + Handler func(...exp.Expression) exp.Expression + HandlerE func(...exp.Expression) (exp.Expression, error) } ) var ( - ref2exp = map[string]*exprHandler{ + ref2exp = ExprHandlerMap{ // keywords "null": { Handler: func(args ...exp.Expression) exp.Expression { @@ -167,31 +172,55 @@ var ( // Result: wrapRes("Number"), // Handler: makeGenericAggFncHandler("AVG"), //}, - // - //// - filtering - //"now": { - // Result: wrapRes("DateTime"), - // Handler: makeGenericFilterFncHandler("NOW"), - //}, - //"quarter": { - // Args: collectParams(true, "DateTime"), - // Result: wrapRes("Number"), - // Handler: makeGenericFilterFncHandler("QUARTER"), - //}, - //"year": { - // Args: collectParams(true, "DateTime"), - // Result: wrapRes("Number"), - // Handler: makeGenericFilterFncHandler("YEAR"), - //}, - //"month": { - // Args: collectParams(true, "DateTime"), - // Result: wrapRes("Number"), - // Handler: makeGenericFilterFncHandler("MONTH"), - //}, - //"date": { - // Args: collectParams(true, "DateTime"), - // Result: wrapRes("Number"), - // Handler: makeGenericFilterFncHandler("DAY"), - //}, + + // - filtering + "now": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("NOW") + }, + }, + "quarter": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("QUARTER", args[0]) + }, + }, + "year": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("YEAR", args[0]) + }, + }, + "month": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("MONTH", args[0]) + }, + }, + "date": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewSQLFunctionExpression("DAY", args[0]) + }, + }, } ) + +func (ee ExprHandlerMap) ExprHandlers() (out ExprHandlerMap) { + out = ref2exp + if out == nil { + out = make(map[string]*ExprHandler) + } + for name, expr := range ee { + out[name] = expr + } + return +} + +func (ee ExprHandlerMap) RefHandler(n *ql.ASTNode, args ...exp.Expression) (exp.Expression, error) { + if ref2exp[n.Ref] == nil { + return nil, fmt.Errorf("unknown ref %q", n.Ref) + } + + if ref2exp[n.Ref].Handler != nil { + return ref2exp[n.Ref].Handler(args...), nil + } + + return ref2exp[n.Ref].HandlerE(args...) +} diff --git a/tests/dal/dal_utils.go b/tests/dal/dal_utils_test.go similarity index 100% rename from tests/dal/dal_utils.go rename to tests/dal/dal_utils_test.go diff --git a/tests/dal/helper.go b/tests/dal/helper.go deleted file mode 100644 index 08cfff0a2..000000000 --- a/tests/dal/helper.go +++ /dev/null @@ -1,238 +0,0 @@ -package dal - -import ( - "context" - "testing" - "time" - - composeTypes "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/auth" - "github.com/cortezaproject/corteza-server/pkg/dal" - "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" - "github.com/cortezaproject/corteza-server/pkg/id" - "github.com/cortezaproject/corteza-server/pkg/rbac" - "github.com/cortezaproject/corteza-server/store" - "github.com/cortezaproject/corteza-server/system/service" - "github.com/cortezaproject/corteza-server/system/types" - "github.com/cortezaproject/corteza-server/tests/helpers" - "github.com/steinfletcher/apitest" - "github.com/stretchr/testify/require" -) - -type ( - helper struct { - t *testing.T - b *testing.B - a *require.Assertions - - cUser *types.User - roleID uint64 - token []byte - } -) - -var ( - testUser *types.User -) - -func newHelperT(t *testing.T) helper { - h := newHelper(t, require.New(t)) - h.t = t - return h -} - -func newHelperB(b *testing.B) helper { - h := newHelper(b, require.New(b)) - h.b = b - return h -} - -func newHelper(t require.TestingT, a *require.Assertions) helper { - var ( - h = helper{ - roleID: id.Next(), - a: a, - } - ctx = context.Background() - - err error - ) - - if testUser == nil { - testUser = &types.User{ - Handle: "test_user", - Name: "test_user", - ID: id.Next(), - } - - err = store.CreateUser(ctx, service.DefaultStore, testUser) - if err != nil { - panic(err) - } - - } - h.cUser = testUser - - h.cUser.SetRoles(h.roleID) - helpers.UpdateRBAC(h.roleID) - h.mockPermissionsWithAccess() - - h.token, err = auth.TokenIssuer.Issue(ctx, auth.WithIdentity(h.cUser)) - if err != nil { - panic(err) - } - - return h -} - -// apitest basics, initialize, set handler, add auth -func (h helper) apiInit() *apitest.APITest { - InitTestApp() - - return apitest. - New(). - Handler(r). - Intercept(helpers.ReqHeaderRawAuthBearer(h.token)) -} - -func (h helper) MyRole() uint64 { - return h.roleID -} - -// Returns context w/ security details -func (h helper) secCtx() context.Context { - return auth.SetIdentityToContext(context.Background(), h.cUser) -} - -func (h helper) mockPermissions(rules ...*rbac.Rule) { - h.a.NoError(rbac.Global().Grant( - // TestService we use does not have any backend storage, - context.Background(), - rules..., - )) -} - -// Prepends allow access rule for system service for everyone -func (h helper) mockPermissionsWithAccess(rules ...*rbac.Rule) { - h.mockPermissions(rules...) -} - -// // // // // // // // // // // // // // // // // // // // // // // // // -// Resource utilities - -func (h helper) createNamespace(name string) *composeTypes.Namespace { - ns := &composeTypes.Namespace{Name: name, Slug: name} - ns.ID = id.Next() - ns.CreatedAt = time.Now() - h.a.NoError(store.CreateComposeNamespace(context.Background(), service.DefaultStore, ns)) - return ns -} - -func (h helper) createSensitivityLevel(res *types.DalSensitivityLevel) *types.DalSensitivityLevel { - if res.ID == 0 { - res.ID = id.Next() - } - - if res.CreatedAt.IsZero() { - res.CreatedAt = time.Now() - } - - h.a.NoError(service.DefaultStore.CreateDalSensitivityLevel(context.Background(), res)) - h.a.NoError(service.DefaultDalSensitivityLevel.ReloadSensitivityLevels(context.Background(), service.DefaultStore)) - return res -} - -func (h helper) createDalConnection(res *types.DalConnection) *types.DalConnection { - if res.ID == 0 { - res.ID = id.Next() - } - - if res.Name == "" { - res.Name = "Test Connection" - } - if res.Handle == "" { - res.Handle = "test_connection" - } - if res.Type == "" { - res.Type = types.DalConnectionResourceType - } - if res.Ownership == "" { - res.Ownership = "tester" - } - - if res.Config.DefaultModelIdent == "" { - res.Config.DefaultModelIdent = "compose_records" - } - if res.Config.DefaultAttributeIdent == "" { - res.Config.DefaultAttributeIdent = "values" - } - if res.Config.DefaultPartitionFormat == "" { - res.Config.DefaultPartitionFormat = "compose_records_{{namespace}}_{{module}}" - } - if res.Config.PartitionFormatValidator == "" { - res.Config.PartitionFormatValidator = "" - } - if res.Config.Connection.Params == nil { - res.Config.Connection = dal.NewDSNConnection("sqlite3://file::memory:?cache=shared&mode=memory") - } - - if len(res.Capabilities.Enforced) == 0 { - res.Capabilities.Enforced = capabilities.FullCapabilities() - } - - if len(res.Capabilities.Supported) == 0 { - res.Capabilities.Supported = capabilities.Set{} - } - - if len(res.Capabilities.Unsupported) == 0 { - res.Capabilities.Unsupported = capabilities.Set{} - } - - if len(res.Capabilities.Enabled) == 0 { - res.Capabilities.Enabled = capabilities.Set{} - } - - if res.CreatedAt.IsZero() { - res.CreatedAt = time.Now() - } - if res.CreatedBy == 0 { - res.CreatedBy = h.cUser.ID - } - - h.a.NoError(service.DefaultStore.CreateDalConnection(context.Background(), res)) - h.a.NoError(service.DefaultDalConnection.ReloadConnections(context.Background())) - return res -} - -func (h helper) getPrimaryConnection() *types.DalConnection { - cc, _, err := store.SearchDalConnections(context.Background(), service.DefaultStore, types.DalConnectionFilter{Type: types.DalPrimaryConnectionResourceType}) - h.a.NoError(err) - - if len(cc) != 1 { - h.a.FailNow("invalid state: no or too many primary connections") - } - - return cc[0] -} - -func makeConnectionDefinition(dsn string) *types.DalConnection { - return &types.DalConnection{ - ID: id.Next(), - Type: types.DalConnectionResourceType, - Config: types.ConnectionConfig{ - DefaultModelIdent: "compose_record", - DefaultAttributeIdent: "values", - - DefaultPartitionFormat: "compose_record_{{namespace}}_{{module}}", - - PartitionFormatValidator: "", - - Connection: dal.NewDSNConnection(dsn), - }, - Capabilities: types.ConnectionCapabilities{ - Supported: capabilities.FullCapabilities(), - }, - } -} - -// // // // // // // // // // // // // // // // // // // // // // // // // diff --git a/tests/dal/main_test.go b/tests/dal/main_test.go index 089f30eed..2b4343691 100644 --- a/tests/dal/main_test.go +++ b/tests/dal/main_test.go @@ -5,10 +5,18 @@ import ( "context" "encoding/json" "fmt" + "github.com/cortezaproject/corteza-server/pkg/auth" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" + "github.com/cortezaproject/corteza-server/pkg/id" + "github.com/cortezaproject/corteza-server/pkg/rbac" + "github.com/cortezaproject/corteza-server/store" + "github.com/steinfletcher/apitest" "io/ioutil" "os" "path" "testing" + "time" "github.com/cortezaproject/corteza-server/app" "github.com/cortezaproject/corteza-server/auth/handlers" @@ -35,10 +43,23 @@ import ( "golang.org/x/text/language" ) +type ( + helper struct { + t *testing.T + b *testing.B + a *require.Assertions + + cUser *types.User + roleID uint64 + token []byte + } +) + var ( - testApp *app.CortezaApp - r chi.Router - hh *handlers.AuthHandlers + testApp *app.CortezaApp + r chi.Router + testUser *types.User + hh *handlers.AuthHandlers ) func init() { @@ -102,6 +123,211 @@ func InitTestApp() { } } +func newHelperT(t *testing.T) helper { + h := newHelper(t, require.New(t)) + h.t = t + return h +} + +func newHelperB(b *testing.B) helper { + h := newHelper(b, require.New(b)) + h.b = b + return h +} + +func newHelper(_ require.TestingT, a *require.Assertions) helper { + var ( + h = helper{ + a: a, + roleID: id.Next(), + } + + ctx = context.Background() + ) + + if testUser == nil { + testUser = &types.User{ + Handle: "test_user", + Name: "test_user", + ID: id.Next(), + } + + err := store.CreateUser(ctx, service.DefaultStore, testUser) + if err != nil { + panic(err) + } + + } + h.cUser = testUser + + h.cUser.SetRoles(h.roleID) + helpers.UpdateRBAC(h.roleID) + h.identityToHelper(ctx, h.cUser) + h.mockPermissionsWithAccess() + + return h +} + +// apiInit basics, initialize, set handler, add auth +func (h helper) apiInit() *apitest.APITest { + InitTestApp() + + return apitest. + New(). + Handler(r). + Intercept(helpers.ReqHeaderRawAuthBearer(h.token)) +} + +func (h helper) MyRole() uint64 { + return h.roleID +} + +// Returns context w/ security details +func (h helper) secCtx() context.Context { + return auth.SetIdentityToContext(context.Background(), h.cUser) +} + +func (h *helper) identityToHelper(ctx context.Context, u *types.User) { + var err error + h.cUser = u + + h.token, err = auth.TokenIssuer.Issue(ctx, auth.WithIdentity(h.cUser)) + if err != nil { + panic(err) + } +} + +func (h helper) mockPermissions(rules ...*rbac.Rule) { + h.a.NoError(rbac.Global().Grant( + // TestService we use does not have any backend storage, + context.Background(), + rules..., + )) +} + +// Prepends allow access rule for system service for everyone +func (h helper) mockPermissionsWithAccess(rules ...*rbac.Rule) { + h.mockPermissions(rules...) +} + +// // // // // // // // // // // // // // // // // // // // // // // // // +// Resource utilities + +func (h helper) createNamespace(name string) *composeTypes.Namespace { + ns := &composeTypes.Namespace{Name: name, Slug: name} + ns.ID = id.Next() + ns.CreatedAt = time.Now() + h.a.NoError(store.CreateComposeNamespace(context.Background(), service.DefaultStore, ns)) + return ns +} + +func (h helper) createSensitivityLevel(res *types.DalSensitivityLevel) *types.DalSensitivityLevel { + if res.ID == 0 { + res.ID = id.Next() + } + + if res.CreatedAt.IsZero() { + res.CreatedAt = time.Now() + } + + h.a.NoError(service.DefaultStore.CreateDalSensitivityLevel(context.Background(), res)) + h.a.NoError(service.DefaultDalSensitivityLevel.ReloadSensitivityLevels(context.Background(), service.DefaultStore)) + return res +} + +func (h helper) createDalConnection(res *types.DalConnection) *types.DalConnection { + if res.ID == 0 { + res.ID = id.Next() + } + + if res.Name == "" { + res.Name = "Test Connection" + } + if res.Handle == "" { + res.Handle = "test_connection" + } + if res.Type == "" { + res.Type = types.DalConnectionResourceType + } + if res.Ownership == "" { + res.Ownership = "tester" + } + + if res.Config.DefaultModelIdent == "" { + res.Config.DefaultModelIdent = "compose_records" + } + if res.Config.DefaultAttributeIdent == "" { + res.Config.DefaultAttributeIdent = "values" + } + if res.Config.DefaultPartitionFormat == "" { + res.Config.DefaultPartitionFormat = "compose_records_{{namespace}}_{{module}}" + } + if res.Config.PartitionFormatValidator == "" { + res.Config.PartitionFormatValidator = "" + } + if res.Config.Connection.Params == nil { + res.Config.Connection = dal.NewDSNConnection("sqlite3://file::memory:?cache=shared&mode=memory") + } + + if len(res.Capabilities.Enforced) == 0 { + res.Capabilities.Enforced = capabilities.FullCapabilities() + } + + if len(res.Capabilities.Supported) == 0 { + res.Capabilities.Supported = capabilities.Set{} + } + + if len(res.Capabilities.Unsupported) == 0 { + res.Capabilities.Unsupported = capabilities.Set{} + } + + if len(res.Capabilities.Enabled) == 0 { + res.Capabilities.Enabled = capabilities.Set{} + } + + if res.CreatedAt.IsZero() { + res.CreatedAt = time.Now() + } + if res.CreatedBy == 0 { + res.CreatedBy = h.cUser.ID + } + + h.a.NoError(service.DefaultStore.CreateDalConnection(context.Background(), res)) + h.a.NoError(service.DefaultDalConnection.ReloadConnections(context.Background())) + return res +} + +func (h helper) getPrimaryConnection() *types.DalConnection { + cc, _, err := store.SearchDalConnections(context.Background(), service.DefaultStore, types.DalConnectionFilter{Type: types.DalPrimaryConnectionResourceType}) + h.a.NoError(err) + + if len(cc) != 1 { + h.a.FailNow("invalid state: no or too many primary connections") + } + + return cc[0] +} + +func makeConnectionDefinition(dsn string) *types.DalConnection { + return &types.DalConnection{ + ID: id.Next(), + Type: types.DalConnectionResourceType, + Config: types.ConnectionConfig{ + DefaultModelIdent: "compose_record", + DefaultAttributeIdent: "values", + + DefaultPartitionFormat: "compose_record_{{namespace}}_{{module}}", + + PartitionFormatValidator: "", + + Connection: dal.NewDSNConnection(dsn), + }, + Capabilities: types.ConnectionCapabilities{ + Supported: capabilities.FullCapabilities(), + }, + } +} + // // // // // // // // // // // // // // // // // // // // // // // // // // Utilities