From a109335c08c823fc34d83a88682b9a7ffd8964a9 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Wed, 27 Apr 2022 13:56:45 +0200 Subject: [PATCH] Searching & filtering support in CRS --- compose/crs/driver.go | 4 +- store/adapters/rdbms/crs/driver.go | 151 +++++++++----- store/adapters/rdbms/crs/driver_test.go | 86 +++++++- store/adapters/rdbms/crs/fetcher.go | 5 - store/adapters/rdbms/crs/fetcher_test.go | 13 +- store/adapters/rdbms/dialect.go | 23 +- store/adapters/rdbms/drivers/mysql/ql.go | 9 + store/adapters/rdbms/drivers/postgres/ql.go | 29 +++ .../adapters/rdbms/drivers/sqlite/dialect.go | 2 +- store/adapters/rdbms/drivers/sqlite/ql.go | 101 +++++++++ store/adapters/rdbms/filters.gen.go | 3 +- store/adapters/rdbms/ql/converter.go | 111 ++++++++++ store/adapters/rdbms/ql/converter_test.go | 75 +++++++ store/adapters/rdbms/ql/ref.go | 197 ++++++++++++++++++ 14 files changed, 732 insertions(+), 77 deletions(-) create mode 100644 store/adapters/rdbms/drivers/mysql/ql.go create mode 100644 store/adapters/rdbms/drivers/postgres/ql.go create mode 100644 store/adapters/rdbms/drivers/sqlite/ql.go create mode 100644 store/adapters/rdbms/ql/converter.go create mode 100644 store/adapters/rdbms/ql/converter_test.go create mode 100644 store/adapters/rdbms/ql/ref.go diff --git a/compose/crs/driver.go b/compose/crs/driver.go index a3628dbcc..302ffb823 100644 --- a/compose/crs/driver.go +++ b/compose/crs/driver.go @@ -66,10 +66,10 @@ type ( // probably somewhere else (on the db level? Getter interface { - GetValue(name string, pos int) (expr.TypedValue, error) + GetValue(name string, pos int) (any, error) } Setter interface { - SetValue(name string, pos int, value expr.TypedValue) error + SetValue(name string, pos int, value any) error } ) diff --git a/store/adapters/rdbms/crs/driver.go b/store/adapters/rdbms/crs/driver.go index bcdc415cc..2f7661cbb 100644 --- a/store/adapters/rdbms/crs/driver.go +++ b/store/adapters/rdbms/crs/driver.go @@ -2,14 +2,18 @@ package crs import ( "context" - "database/sql" + "fmt" + "strings" "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/data" "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/qlng" "github.com/cortezaproject/corteza-server/store/adapters/rdbms" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" + "github.com/jmoiron/sqlx" ) type ( @@ -18,9 +22,8 @@ type ( } connection interface { - ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) - QueryRowContext(context.Context, string, ...interface{}) *sql.Row - QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + sqlx.QueryerContext + sqlx.ExecerContext } attrExpression interface { @@ -42,10 +45,15 @@ type ( decode func([]*data.Attribute, any, *types.Record) error } + queryParser interface { + Parse(string) (exp.Expression, error) + } + crs struct { model *data.Model + conn connection - conn connection + queryParser queryParser deepJsonFn func(ident string, pp ...any) exp.LiteralExpression @@ -69,11 +77,13 @@ type ( // // It abstracts database table and its columns and provides unified interface // for fetching and storing records. -func CRS(m *data.Model, conn connection) *crs { +func CRS(m *data.Model, c connection) *crs { var ( ms = &crs{ model: m, - conn: conn, + conn: c, + + queryParser: ql.Converter(), } ) @@ -82,6 +92,12 @@ func CRS(m *data.Model, conn connection) *crs { _ = ms.genColumns() + ms.queryParser = ql.Converter( + ql.SymHandler(func(node *qlng.ASTNode) (exp.Expression, error) { + return ms.attrToExpr(node.Symbol) + }), + ) + return ms } @@ -173,6 +189,16 @@ func (ms *crs) genColumns() error { return nil } +func (ms *crs) Truncate(ctx context.Context) error { + sql, args, err := ms.truncateSql().ToSQL() + if err != nil { + return err + } + + _, err = ms.conn.ExecContext(ctx, sql, args...) + return err +} + func (ms *crs) Create(ctx context.Context, rr ...*types.Record) error { sql, args, err := ms.insertSql(rr...).ToSQL() if err != nil { @@ -204,27 +230,9 @@ func (ms *crs) Delete(ctx context.Context, r *types.Record) error { } func (ms *crs) Search(ctx context.Context, f types.RecordFilter) (i *iterator, err error) { - // @todo apply filter - // @todo apply cursor constraints - - if f.PageCursor != nil { - // Page cursor exists; we need to validate it against used sort - // To cover the case when paging cursor is set but sorting is empty, we collect the sorting instructions - // from the cursor. - // This (extracted sorting info) is then returned as part of response - if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { - return - } - } - - // Make sure results are always sorted at least by primary key - if f.Sort.Get(sysID) == nil { - f.Sort = append(f.Sort, &filter.SortExpr{ - Column: sysID, - Descending: f.Sort.LastDescending(), - }) - } - + // construct base query + sql := ms.searchSql(f) + _ = sql return &iterator{ms: ms}, nil } @@ -526,9 +534,30 @@ func (ms *crs) Search(ctx context.Context, f types.RecordFilter) (i *iterator, e // constructs SQL for selecting records from a table, converting parts of record filter into conditions func (ms *crs) searchSql(f types.RecordFilter) *goqu.SelectDataset { var ( - cnd []exp.Expression + err error + base = ms.selectSql() + tmp exp.Expression + cnd []exp.Expression ) + if f.PageCursor != nil { + // Page cursor exists; we need to validate it against used sort + // To cover the case when paging cursor is set but sorting is empty, we collect the sorting instructions + // from the cursor. + // This (extracted sorting info) is then returned as part of response + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return base.SetError(err) + } + } + + if f.Sort.Get(sysID) == nil { + // Make sure results are always sorted at least by primary key + f.Sort = append(f.Sort, &filter.SortExpr{ + Column: sysID, + Descending: f.Sort.LastDescending(), + }) + } + { // Add module & namespace constraints when model expects (has configured attributes) for them // @@ -541,16 +570,22 @@ func (ms *crs) searchSql(f types.RecordFilter) *goqu.SelectDataset { if ms.sysExprNamespaceID != nil { cnd = append(cnd, ms.sysExprNamespaceID.Eq(f.NamespaceID)) + } else { + // @todo check if f.NamespaceID is compatible } if ms.sysExprModuleID != nil { cnd = append(cnd, ms.sysExprModuleID.Eq(f.ModuleID)) + } else { + // @todo check if f.ModuleID is compatible } } { - // Limit by LabeledIDs (list of record IDs) - cnd = append(cnd, ms.sysColumnID.In(f.LabeledIDs)) + if len(f.LabeledIDs) > 0 { + // Limit by LabeledIDs (list of record IDs) + cnd = append(cnd, ms.sysColumnID.In(f.LabeledIDs)) + } } { @@ -569,7 +604,13 @@ func (ms *crs) searchSql(f types.RecordFilter) *goqu.SelectDataset { } } - // @todo ql (AST) => []exp.Expression + if len(strings.TrimSpace(f.Query)) > 0 { + if tmp, err = ms.queryParser.Parse(f.Query); err != nil { + return base.SetError(err) + } + + cnd = append(cnd, tmp) + } return ms.selectSql().Where(cnd...) } @@ -593,6 +634,10 @@ func (ms *crs) selectSql() *goqu.SelectDataset { return q } +func (ms *crs) truncateSql() (_ *goqu.TruncateDataset) { + return goqu.Truncate(ms.table) +} + func (ms *crs) insertSql(rr ...*types.Record) (_ *goqu.InsertDataset) { var ( err error @@ -631,24 +676,24 @@ func (ms *crs) deleteByIdSql(id uint64) *goqu.DeleteDataset { return goqu.Delete(ms.table).Where(ms.sysColumnID.Eq(id)) } -//func (ms *crs) attrToExpr(ident string) (exp.Comparable, bool) { -// if !ms.model.HasAttribute(ident) { -// return nil, false -// } -// -// attr := ms.model.Attributes.FindByIdent(ident) -// switch s := attr.Store.(type) { -// case *data.StoreCodecAlias: -// // using column directly -// return exp.NewIdentifierExpression("", ms.model.Ident, s.Ident), true -// -// case *data.StoreCodecStdRecordValueJSON: -// // using JSON to handle embedded values -// return ms.deepJsonFn(ms.model.Ident, s.Ident, 0), true -// } -// -// return exp.NewIdentifierExpression("", ms.model.Ident, ident), true -//} +func (ms *crs) attrToExpr(ident string) (exp.LiteralExpression, error) { + if !ms.model.HasAttribute(ident) { + return nil, fmt.Errorf("unknown attribute %q", ident) + } + + attr := ms.model.Attributes.FindByIdent(ident) + switch s := attr.Store.(type) { + case *data.StoreCodecAlias: + // using column directly + return exp.NewLiteralExpression(fmt.Sprintf("%q.%q", ms.model.Ident, s.Ident)), nil + + case *data.StoreCodecStdRecordValueJSON: + // using JSON to handle embedded values + return ms.deepJsonFn(s.Ident, attr.Ident, 0), nil + } + + return exp.NewLiteralExpression(fmt.Sprintf("%q.%q", ms.model.Ident, ident)), nil +} func attrColumnIdent(att *data.Attribute) string { switch ss := att.Store.(type) { @@ -683,14 +728,14 @@ func attrColumnIdent(att *data.Attribute) string { // return //} -// -//func valueExpr(att *data.Attribute) (exp.Expression, error) { +// attributeExp converts attribute to sql expression +//func attributeExp(att *data.Attribute) (exp.Expression, error) { // if att.Store == nil { // return exp.NewIdentifierExpression("", "", att.Ident), nil // // } // switch ss := att.Store.(type) { -// case data.StoreCodecJSON: +// case data.StoreCodecStdRecordValueJSON: // return exp.NewIdentifierExpression("", "", ss.Ident), nil // case data.StoreCodecAlias: // return exp.NewIdentifierExpression("", "", ss.Ident).As(att.Ident), nil diff --git a/store/adapters/rdbms/crs/driver_test.go b/store/adapters/rdbms/crs/driver_test.go index 6b3104ef8..72c74a074 100644 --- a/store/adapters/rdbms/crs/driver_test.go +++ b/store/adapters/rdbms/crs/driver_test.go @@ -6,6 +6,7 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/data" + _ "github.com/doug-martin/goqu/v9/dialect/postgres" "github.com/stretchr/testify/require" ) @@ -22,7 +23,6 @@ func Test_sqlizers(t *testing.T) { ms = CRS(m, nil) - req = require.New(t) q sqlizer sql string args []any @@ -67,6 +67,7 @@ func Test_sqlizers(t *testing.T) { for _, c := range cases { t.Run("", func(t *testing.T) { + req := require.New(t) q = c.fn() sql, args, err = q.ToSQL() @@ -81,5 +82,86 @@ func Test_sqlizers(t *testing.T) { }) } - +} + +func Test_search(t *testing.T) { + var ( + m = &data.Model{ + Ident: "test_tbl", + Attributes: data.AttributeSet{ + &data.Attribute{Ident: sysID, Type: &data.TypeID{}, Store: &data.StoreCodecAlias{Ident: "id"}}, + &data.Attribute{Ident: sysModuleID, Type: &data.TypeID{}, Store: &data.StoreCodecAlias{Ident: "rel_module"}}, + &data.Attribute{Ident: sysCreatedAt, Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "created_at"}}, + &data.Attribute{Ident: sysUpdatedAt, Type: &data.TypeTimestamp{}, Store: &data.StoreCodecAlias{Ident: "updated_at"}}, + &data.Attribute{Ident: "foo", Type: &data.TypeText{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, + &data.Attribute{Ident: "bar", Type: &data.TypeNumber{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, + &data.Attribute{Ident: "baz", Type: &data.TypeBoolean{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, + &data.Attribute{Ident: "bbb", Type: &data.TypeUUID{}, Store: &data.StoreCodecStdRecordValueJSON{Ident: "meta"}}, + &data.Attribute{Ident: "phy", Type: &data.TypeText{}, Store: &data.StoreCodecPlain{}}, + }, + } + + sql string + args []any + err error + + moduleID = uint64(1<<64 - 1) + + prefix = `SELECT "test_tbl"."id", "test_tbl"."rel_module", "test_tbl"."created_at", "test_tbl"."updated_at", "test_tbl"."meta", "test_tbl"."phy" FROM "test_tbl"` + ) + + cases := []struct { + f types.RecordFilter + sql string + args []any + err error + }{ + { + f: types.RecordFilter{ + ModuleID: moduleID, + }, + sql: prefix + ` WHERE ("test_tbl"."rel_module" = $1)`, + args: []any{moduleID}, + err: nil, + }, + { + f: types.RecordFilter{ + ModuleID: moduleID, + Query: "moduleID == 1234", + }, + sql: prefix + ` WHERE (("test_tbl"."rel_module" = $1) AND ("test_tbl"."rel_module" = $2))`, + args: []any{moduleID, int64(1234)}, + err: nil, + }, + { + f: types.RecordFilter{ + ModuleID: moduleID, + Query: `bar = 1 AND foo = phy`, + }, + sql: prefix + ` WHERE (("test_tbl"."rel_module" = $1) AND (("meta"->'bar'->0 = $2) AND ("meta"->'foo'->0 = "test_tbl"."phy")))`, + args: []any{moduleID, int64(1)}, + err: nil, + }, + } + + d := CRS(m, nil) + + for _, c := range cases { + t.Run("", func(t *testing.T) { + req := require.New(t) + q := d.searchSql(c.f).WithDialect("postgres") + sql, args, err = q.ToSQL() + if c.args == nil { + req.NoError(err) + } else { + req.ErrorIs(err, c.err) + } + + if len(c.sql) > 0 { + req.Equal(c.sql, sql) + } + + req.Equal(c.args, args) + }) + } } diff --git a/store/adapters/rdbms/crs/fetcher.go b/store/adapters/rdbms/crs/fetcher.go index bbbe8f4ac..565aae746 100644 --- a/store/adapters/rdbms/crs/fetcher.go +++ b/store/adapters/rdbms/crs/fetcher.go @@ -6,7 +6,6 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/data" - "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/google/uuid" ) @@ -107,7 +106,3 @@ func newScanBuffer(d *crs) []any { return vv } - -func CursorBackward(r *types.Record, sort filter.SortExprSet) *filter.PagingCursor { return nil } -func CursorForward(r *types.Record, sort filter.SortExprSet) *filter.PagingCursor { return nil } -func Drain(*iterator) ([]*types.Record, error) { return nil, nil } diff --git a/store/adapters/rdbms/crs/fetcher_test.go b/store/adapters/rdbms/crs/fetcher_test.go index 371059c02..dbb091910 100644 --- a/store/adapters/rdbms/crs/fetcher_test.go +++ b/store/adapters/rdbms/crs/fetcher_test.go @@ -13,12 +13,12 @@ import ( "github.com/cortezaproject/corteza-server/store/adapters/rdbms/drivers/sqlite" ) -const ( - repeatReads = 1000 - totalRecords = 100 -) +func TestIterator(t *testing.T) { + const ( + repeatReads = 1000 + totalRecords = 100 + ) -func TestIteratorNG(t *testing.T) { ctx := context.Background() ctx = logger.ContextWithValue(ctx, logger.MakeDebugLogger()) cfg, err := sqlite.NewConfig("sqlite3://file::memory:?cache=shared&mode=memory") @@ -60,8 +60,7 @@ func TestIteratorNG(t *testing.T) { } d := CRS(m, db) - _, err = db.Exec("DELETE FROM test_tbl") - if err != nil { + if err = d.Truncate(ctx); err != nil { t.Errorf("could not truncate: %v", err) } diff --git a/store/adapters/rdbms/dialect.go b/store/adapters/rdbms/dialect.go index 9b10edf44..ac9e6477a 100644 --- a/store/adapters/rdbms/dialect.go +++ b/store/adapters/rdbms/dialect.go @@ -6,11 +6,17 @@ package rdbms import ( "fmt" + "strconv" "strings" + "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" ) +func init() { + goqu.SetDefaultPrepared(true) +} + // DeepIdentJSON constructs expression with chain of JSON operators // that point value inside JSON document // @@ -25,21 +31,26 @@ import ( // SQLite // https://www.sqlite.org/json1.html#jptr func DeepIdentJSON(ident string, pp ...any) exp.LiteralExpression { + // strings.ReplaceAll(str, "'", `\'`) + var ( - sql = "?" + strings.Repeat("->?", len(pp)) - args = []any{exp.ParseIdentifier(ident)} + //sql = "?" + strings.Repeat("->?", len(pp)) + //args = []any{exp.ParseIdentifier(ident)} + sql = strconv.Quote(ident) ) for _, p := range pp { - switch p.(type) { - case string, int: - args = append(args, exp.NewLiteralExpression("?", p)) + switch path := p.(type) { + case string: + sql += "->'" + strings.ReplaceAll(path, "'", `\'`) + "'" + case int: + sql += fmt.Sprintf("->%d", path) default: panic("invalid type") } } - return exp.NewLiteralExpression(sql, args...) + return exp.NewLiteralExpression(sql) } // JsonPath constructs json-path string from the slice of path parts. diff --git a/store/adapters/rdbms/drivers/mysql/ql.go b/store/adapters/rdbms/drivers/mysql/ql.go new file mode 100644 index 000000000..4004a48e7 --- /dev/null +++ b/store/adapters/rdbms/drivers/mysql/ql.go @@ -0,0 +1,9 @@ +package mysql + +//var ( +// sqlExprRegistry = map[string]rdbms.HandlerSig{} +//) +// +//func sqlASTFormatter(n *qlng.ASTNode) rdbms.HandlerSig { +// return sqlExprRegistry[n.Ref] +//} diff --git a/store/adapters/rdbms/drivers/postgres/ql.go b/store/adapters/rdbms/drivers/postgres/ql.go new file mode 100644 index 000000000..e69c68311 --- /dev/null +++ b/store/adapters/rdbms/drivers/postgres/ql.go @@ -0,0 +1,29 @@ +package postgres + +//var ( +// sqlExprRegistry = map[string]rdbms.HandlerSig{ +// // functions +// // - filtering +// "quarter": makeGenericExtrFncHandler("QUARTER"), +// "year": makeGenericExtrFncHandler("YEAR"), +// "month": makeGenericExtrFncHandler("MONTH"), +// "date": makeGenericExtrFncHandler("DAY"), +// } +//) +// +//func sqlASTFormatter(n *qlng.ASTNode) rdbms.HandlerSig { +// return sqlExprRegistry[n.Ref] +//} +// +//func makeGenericExtrFncHandler(extr string) rdbms.HandlerSig { +// return func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("EXTRACT(%s FROM %s)", extr, aa[0].S) +// args = aa[0].Args +// return +// } +//} diff --git a/store/adapters/rdbms/drivers/sqlite/dialect.go b/store/adapters/rdbms/drivers/sqlite/dialect.go index 25023b531..5a65fa32b 100644 --- a/store/adapters/rdbms/drivers/sqlite/dialect.go +++ b/store/adapters/rdbms/drivers/sqlite/dialect.go @@ -8,7 +8,7 @@ import ( func init() { d := dialect.DialectOptions() - // https://github.com/doug-martin/goqu/pull/330 + // https://github.com/doug-martin/goqu/v9/pull/330 d.TruncateClause = []byte("DELETE FROM") // Overriding vanila SQLite dialect diff --git a/store/adapters/rdbms/drivers/sqlite/ql.go b/store/adapters/rdbms/drivers/sqlite/ql.go new file mode 100644 index 000000000..705fc2e9c --- /dev/null +++ b/store/adapters/rdbms/drivers/sqlite/ql.go @@ -0,0 +1,101 @@ +package sqlite + +//var ( +//sqlExprRegistry = map[string]rdbms.HandlerSig{ +// // functions +// // - filtering +// "now": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// if len(aa) != 0 { +// err = fmt.Errorf("expecting 0 arguments, got %d", len(aa)) +// return +// } +// +// out = "DATE('now')" +// return +// }, +// "quarter": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("(CAST(STRFTIME('%%m', %s) AS INTEGER) + 2) / 3", aa[0].S) +// args = aa[0].Args +// return +// }, +// "year": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("STRFTIME('%%Y', %s)", aa[0].S) +// args = aa[0].Args +// return +// }, +// "month": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("STRFTIME('%%m', %s)", aa[0].S) +// args = aa[0].Args +// return +// }, +// "date": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 arguments, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("STRFTIME('%%d', %s)", aa[0].S) +// args = aa[0].Args +// return +// }, +// +// // - strings +// "concat": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// selfEnclosed = true +// +// params := make([]string, len(aa)) +// for i, a := range aa { +// params[i] = a.S +// args = append(args, a.Args...) +// } +// +// out = fmt.Sprintf("(%s)", strings.Join(params, "||")) +// return +// }, +// +// // - typecast +// "float": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// selfEnclosed = true +// +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 argument, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("CAST(%s AS FLOAT)", aa[0].S) +// args = aa[0].Args +// return +// }, +// "string": func(aa ...rdbms.FormattedASTArgs) (out string, args []interface{}, selfEnclosed bool, err error) { +// selfEnclosed = true +// +// if len(aa) != 1 { +// err = fmt.Errorf("expecting 1 argument, got %d", len(aa)) +// return +// } +// +// out = fmt.Sprintf("CAST(%s AS TEXT)", aa[0].S) +// args = aa[0].Args +// return +// }, +//} +//) +// +//func sqlASTFormatter(n *qlng.ASTNode) rdbms.HandlerSig { +// return sqlExprRegistry[n.Ref] +//} diff --git a/store/adapters/rdbms/filters.gen.go b/store/adapters/rdbms/filters.gen.go index addd3b842..e6f0cd674 100644 --- a/store/adapters/rdbms/filters.gen.go +++ b/store/adapters/rdbms/filters.gen.go @@ -7,6 +7,8 @@ package rdbms // import ( + "strings" + automationType "github.com/cortezaproject/corteza-server/automation/types" composeType "github.com/cortezaproject/corteza-server/compose/types" federationType "github.com/cortezaproject/corteza-server/federation/types" @@ -17,7 +19,6 @@ import ( rbacType "github.com/cortezaproject/corteza-server/pkg/rbac" systemType "github.com/cortezaproject/corteza-server/system/types" "github.com/doug-martin/goqu/v9" - "strings" ) type ( diff --git a/store/adapters/rdbms/ql/converter.go b/store/adapters/rdbms/ql/converter.go new file mode 100644 index 000000000..bbed4637d --- /dev/null +++ b/store/adapters/rdbms/ql/converter.go @@ -0,0 +1,111 @@ +package ql + +import ( + "fmt" + + "github.com/cortezaproject/corteza-server/pkg/qlng" + "github.com/doug-martin/goqu/v9/exp" +) + +type ( + refHandler func(*qlng.ASTNode, ...exp.Expression) (exp.Expression, error) + symHandler func(*qlng.ASTNode) (exp.Expression, error) + valHandler func(*qlng.ASTNode) (exp.Expression, error) + + // converts query and AST from the parsed query into goqu expression + // that can be used in the SQL + converter struct { + parser *qlng.Parser + refHandler refHandler + symHandler symHandler + valHandler valHandler + } + + op func(*converter) +) + +// Initializes new converter +func Converter(oo ...op) *converter { + c := &converter{ + parser: qlng.NewParser(), + refHandler: DefaultRefHandler, + symHandler: DefaultSymbolHandler, + valHandler: DefaultValueHandler, + } + + for _, o := range oo { + o(c) + } + + return c +} + +// RefHandler sets custom ref handler +func RefHandler(h refHandler) op { + return func(c *converter) { + c.refHandler = h + } +} + +// SymHandler sets custom symbol handler +func SymHandler(h symHandler) op { + return func(c *converter) { + c.symHandler = h + } +} + +// ValHandler sets custom value handler +func ValHandler(h valHandler) op { + return func(c *converter) { + c.valHandler = h + } +} + +func (c *converter) Parse(q string) (exp.Expression, error) { + n, err := c.parser.Parse(q) + if err != nil { + return nil, err + } + + return c.Convert(n) +} + +// Convert recursively walks through AST and converts the entire tree into expression +// that can be used in goqu SQL builder +func (c *converter) Convert(n *qlng.ASTNode) (_ exp.Expression, err error) { + switch { + case n.Symbol != "": + return c.symHandler(n) + case n.Value != nil: + return c.valHandler(n) + } + + args := make([]exp.Expression, len(n.Args)) + for i, a := range n.Args { + args[i], err = c.Convert(a) + if err != nil { + return + } + } + + return c.refHandler(n, args...) +} + +// DefaultRefHandler converts ref from the AST node using ref2exp +func DefaultRefHandler(n *qlng.ASTNode, args ...exp.Expression) (exp.Expression, error) { + if ref2exp[n.Ref] == nil { + return nil, fmt.Errorf("unknown ref %q", n.Ref) + } + + return ref2exp[n.Ref].Handler(args...), nil +} + +// DefaultSymbolHandler parses symbol from the AST node into an identifier +func DefaultSymbolHandler(n *qlng.ASTNode) (exp.Expression, error) { + return exp.ParseIdentifier(n.Symbol), nil +} + +// DefaultValueHandler converts node into placeholder and a new attribute +func DefaultValueHandler(n *qlng.ASTNode) (exp.Expression, error) { + return exp.NewLiteralExpression("?", n.Value.V.Get()), nil +} diff --git a/store/adapters/rdbms/ql/converter_test.go b/store/adapters/rdbms/ql/converter_test.go new file mode 100644 index 000000000..dec8ecd96 --- /dev/null +++ b/store/adapters/rdbms/ql/converter_test.go @@ -0,0 +1,75 @@ +package ql + +import ( + "strings" + "testing" + + "github.com/doug-martin/goqu/v9" + "github.com/stretchr/testify/require" +) + +func TestConverter(t *testing.T) { + const WHERE = " WHERE " + var ( + base = goqu.Dialect("sqlite").Select().Prepared(true) + conv = Converter() + + cases = []struct { + qry string + sql string + args []any + }{ + { + qry: `foo = 1`, + sql: `("foo" = ?)`, + args: []any{int64(1)}, + }, + { + qry: `foo = 1 AND bar = baz`, + sql: `(("foo" = ?) AND ("bar" = "baz"))`, + args: []any{int64(1)}, + }, + { + qry: `foo = 1 OR (bar = 'baz')`, + sql: `(("foo" = ?) OR ("bar" = ?))`, + args: []any{int64(1), "baz"}, + }, + { + qry: `foo is not null`, + sql: `("foo" IS NOT NULL)`, + args: []any{}, + }, + { + qry: `!foo`, + sql: `(NOT "foo")`, + args: []any{}, + }, + { + qry: `one + 2 / 3 * 4 < 10`, + sql: `("one" + ? / ? * ? < ?)`, + args: []any{int64(2), int64(3), int64(4), int64(10)}, + }, + } + ) + + for _, c := range cases { + t.Run(c.qry, func(t *testing.T) { + req := require.New(t) + + t.Log(c.qry) + ee, err := conv.Parse(c.qry) + req.NoError(err) + + sql, args, err := base.Where(ee).ToSQL() + req.NoError(err) + + p := strings.Index(sql, WHERE) + req.Positive(p) + + sql = sql[p+len(WHERE):] + + req.Equal(c.sql, sql) + req.Equal(c.args, args) + }) + } +} diff --git a/store/adapters/rdbms/ql/ref.go b/store/adapters/rdbms/ql/ref.go new file mode 100644 index 000000000..e68d5fac5 --- /dev/null +++ b/store/adapters/rdbms/ql/ref.go @@ -0,0 +1,197 @@ +package ql + +import ( + "github.com/doug-martin/goqu/v9/exp" +) + +type ( + exprHandler struct { + Handler func(...exp.Expression) exp.Expression + } +) + +var ( + ref2exp = map[string]*exprHandler{ + // keywords + "null": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("NULL") + }, + }, + "nnull": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("NOT NULL") + }, + }, + + // operators + "not": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("(NOT ?)", args[0]) + }, + }, + + // - bool + "and": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewExpressionList(exp.AndType, args...) + }, + }, + "or": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewExpressionList(exp.OrType, args...) + }, + }, + + // - comp. + "eq": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.EqOp, args[0], args[1]) + }, + }, + "ne": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.NeqOp, args[0], args[1]) + }, + }, + "lt": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.LtOp, args[0], args[1]) + }, + }, + "le": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.LteOp, args[0], args[1]) + }, + }, + "gt": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.GtOp, args[0], args[1]) + }, + }, + "ge": { + //Handler: makeGenericCompHandler(">="), + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.GteOp, args[0], args[1]) + }, + }, + + // - math + "add": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("? + ?", args[0], args[1]) + }, + }, + "sub": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("? - ?", args[0], args[1]) + }, + }, + "mult": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("? * ?", args[0], args[1]) + }, + }, + "div": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("? / ?", args[0], args[1]) + }, + }, + + // - strings + "concat": { + Handler: func(args ...exp.Expression) exp.Expression { + aa := make([]any, len(args)) + for a := range args { + aa[a] = args[a] + } + + return exp.NewSQLFunctionExpression("CONCAT", aa...) + }, + }, + + // @todo better negation? + "like": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.ILikeOp, args[0], args[1]) + }, + }, + "nlike": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.NotILikeOp, args[0], args[1]) + }, + }, + + "is": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.IsOp, args[0], args[1]) + }, + }, + + "nis": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewBooleanExpression(exp.IsNotOp, args[0], args[1]) + }, + }, + + "group": { + Handler: func(args ...exp.Expression) exp.Expression { + return exp.NewLiteralExpression("(?)", args[0]) + }, + }, + + // functions + // - aggregation + //"count": { + // Args: collectParams(false, "Any"), + // Result: wrapRes("Number"), + // Handler: makeGenericAggFncHandler("COUNT"), + //}, + //"sum": { + // Args: collectParams(true, "Any"), + // Result: wrapRes("Number"), + // Handler: makeGenericAggFncHandler("SUM"), + //}, + //"max": { + // Args: collectParams(true, "Any"), + // Result: wrapRes("Number"), + // Handler: makeGenericAggFncHandler("MAX"), + //}, + //"min": { + // Args: collectParams(true, "Any"), + // Result: wrapRes("Number"), + // Handler: makeGenericAggFncHandler("MIN"), + //}, + //"avg": { + // Args: collectParams(true, "Any"), + // Result: wrapRes("Number"), + // Handler: makeGenericAggFncHandler("AVG"), + //}, + // + //// - filtering + //"now": { + // Result: wrapRes("DateTime"), + // Handler: makeGenericFilterFncHandler("NOW"), + //}, + //"quarter": { + // Args: collectParams(true, "DateTime"), + // Result: wrapRes("Number"), + // Handler: makeGenericFilterFncHandler("QUARTER"), + //}, + //"year": { + // Args: collectParams(true, "DateTime"), + // Result: wrapRes("Number"), + // Handler: makeGenericFilterFncHandler("YEAR"), + //}, + //"month": { + // Args: collectParams(true, "DateTime"), + // Result: wrapRes("Number"), + // Handler: makeGenericFilterFncHandler("MONTH"), + //}, + //"date": { + // Args: collectParams(true, "DateTime"), + // Result: wrapRes("Number"), + // Handler: makeGenericFilterFncHandler("DAY"), + //}, + } +)