From 3bf030685bdd640027fe28d9d5635c27d5272f3d Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Sun, 6 Nov 2022 15:56:59 +0100 Subject: [PATCH] Extend RDBMS DAL tests and fix issues with bool & numeric types --- store/adapters/rdbms/dal/main_test.go | 61 ++++++++++- store/adapters/rdbms/dal/model.go | 9 +- store/adapters/rdbms/dal/model_test.go | 103 +++++++++++++++++- .../adapters/rdbms/drivers/sqlite/connect.go | 7 +- .../adapters/rdbms/drivers/sqlite/dialect.go | 6 +- 5 files changed, 173 insertions(+), 13 deletions(-) diff --git a/store/adapters/rdbms/dal/main_test.go b/store/adapters/rdbms/dal/main_test.go index 24d6d1ecf..713f56c57 100644 --- a/store/adapters/rdbms/dal/main_test.go +++ b/store/adapters/rdbms/dal/main_test.go @@ -18,7 +18,8 @@ import ( ) type ( - kv map[string]any + kv map[string]any + kvv map[string][]any ) var ( @@ -92,6 +93,64 @@ func (r kv) String() string { return out } +func (r kvv) Set(k string, v ...any) kvv { + r[k] = v + return r +} + +func (r kvv) CountValues() map[string]uint { + out := make(map[string]uint) + + for k := range r { + out[k] = uint(len(r[k])) + } + + return out +} + +func (r kvv) GetValue(k string, p uint) (any, error) { + if r[k] == nil || len(r[k]) <= int(p) { + return nil, fmt.Errorf("kvv: out of bounds") + } + + return r[k][p], nil +} + +func (r kvv) SetValue(k string, p uint, v any) error { + if r[k] == nil { + r[k] = make([]any, 0, 1) + } else if len(r[k]) < int(p) { + r[k][p] = v + } else { + r[k] = append(r[k], v) + } + + return nil +} + +// String function returns string representation of the kv with sorted keys +func (r kvv) String() string { + // sort keys from map + keys := make([]string, 0, len(r)) + for k := range r { + keys = append(keys, k) + } + + sort.Strings(keys) + + // build string by iterating over sorted keys and appending values + var out string + for i, k := range keys { + if i > 0 { + out += " " + } + + out += fmt.Sprintf("%s=%v", k, r[k]) + } + + return out +} + func qlParse(req *require.Assertions, q string) *ql.ASTNode { n, err := ql.NewParser().Parse(q) req.NoError(err) diff --git a/store/adapters/rdbms/dal/model.go b/store/adapters/rdbms/dal/model.go index 673ce0158..c5359cd7a 100644 --- a/store/adapters/rdbms/dal/model.go +++ b/store/adapters/rdbms/dal/model.go @@ -573,9 +573,14 @@ func (d *model) aggregateSql(f filter.Filter, groupBy []dal.AggregateAttr, out [ converter = ql.Converter( ql.SymHandler(func(node *ql.ASTNode) (exp.Expression, error) { sym := dal.NormalizeAttrNames(node.Symbol) + if a2expr[sym] != nil { - // is aliased expression? - return a2expr[sym], nil + if d.dialect.Nuances().HavingClauseMustUseAlias { + // is aliased expression? + return a2expr[sym], nil + } else { + return exp.NewIdentifierExpression("", "", sym), nil + } } // if not, use the default handler diff --git a/store/adapters/rdbms/dal/model_test.go b/store/adapters/rdbms/dal/model_test.go index 4481b4806..bd19e0ee5 100644 --- a/store/adapters/rdbms/dal/model_test.go +++ b/store/adapters/rdbms/dal/model_test.go @@ -93,6 +93,105 @@ func TestModel_Search(t *testing.T) { req.Equal("group=g0 item=i1000 price=1000 published=1", rows[0].String()) } +// Should be part of general DAL testing (not only RDBMS) +func TestModel_Search2(t *testing.T) { + var ( + req = require.New(t) + + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) + + baseModel = &dal.Model{ + Ident: t.Name(), + Attributes: func() (aa []*dal.Attribute) { + s := &dal.CodecRecordValueSetJSON{Ident: "values"} + aa = []*dal.Attribute{ + {Ident: "phyTxt", Type: &dal.TypeText{}}, + {Ident: "phyNum", Type: &dal.TypeNumber{}}, + {Ident: "phyBool", Type: &dal.TypeBoolean{}}, + {Ident: "jsonEncTxt", Type: &dal.TypeText{}, Store: s}, + {Ident: "jsonEncTxtMV", Type: &dal.TypeText{}, Store: s, MultiValue: true}, + {Ident: "jsonEncNum", Type: &dal.TypeNumber{}, Store: s}, + {Ident: "jsonEncNumMV", Type: &dal.TypeNumber{}, Store: s, MultiValue: true}, + {Ident: "jsonEncBool", Type: &dal.TypeBoolean{}, Store: s}, + {Ident: "jsonEncBoolMV", Type: &dal.TypeBoolean{}, Store: s, MultiValue: true}, + } + + // iterate through all attributsand set Filterable flag to true + for _, a := range aa { + a.Filterable = true + a.Sortable = true + } + + return + }(), + } + + m = Model(baseModel, s.DB, s.Dialect) + + search = func(t *testing.T, f filter.Filter) []kv { + req := require.New(t) + i, err := m.Search(f) + + req.NoError(err) + req.NotNil(i) + + defer req.NoError(i.Close()) + + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) + + rows := make([]kv, 0, 5) + for i.Next(ctx) { + row := kv{} + req.NoError(i.Scan(row)) + rows = append(rows, row) + } + + req.NoError(i.Err()) + return rows + } + ) + + table, err := s.DataDefiner.ConvertModel(baseModel) + req.NoError(err) + + t.Logf("Creating temporary table %q", table.Ident) + table.Temporary = true + req.NoError(s.DataDefiner.TableCreate(ctx, table)) + + { + noLogCtx := context.Background() // no need to log inserts + req.NoError(m.Create(noLogCtx, (&kvv{}). + Set("phyTxt", "bar"). + Set("phyNum", 42). + Set("phyBool", false). + Set("jsonEncTxt", "bar"). + Set("jsonEncTxtMV", "bar", "foo"). + Set("jsonEncNum", 42). + Set("jsonEncNumMV", 21, 42). + Set("jsonEncBool", false). + Set("jsonEncBoolMV", false, true), + )) + } + + req.Len(search(t, filter.Generic(filter.WithExpression("phyTxt = 'bar'"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("phyTxt = 'baz'"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("phyNum = 42"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("phyNum = 21"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("!phyBool"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("phyBool"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncTxt = 'bar'"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncTxt = 'baz'"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncTxtMV = 'bar'"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncTxtMV = 'baz'"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncTxtMV = 'foo'"))), 0, "should not match the second value") + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncNum = 42"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncNum = 21"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncNumMV = 21"))), 1) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncNumMV = 22"))), 0) + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncNumMV = 42"))), 0, "should not match the second value") + req.Len(search(t, filter.Generic(filter.WithExpression("jsonEncBoolMV"))), 0, "should not match the second value") +} + func TestModel_Aggregate(t *testing.T) { _ = logger.Default() @@ -349,7 +448,6 @@ func TestModel_AggregateWithHaving(t *testing.T) { func TestModel_AggregateHavingGroup(t *testing.T) { _ = logger.Default() - var ( req = require.New(t) @@ -412,9 +510,6 @@ func TestModel_AggregateHavingGroup(t *testing.T) { row = kv{} req.NoError(i.Scan(row)) - // due to difference of number of decimal digits in different DBs, we need to do this - // to make sure we get the same result - rows = append(rows, row) } diff --git a/store/adapters/rdbms/drivers/sqlite/connect.go b/store/adapters/rdbms/drivers/sqlite/connect.go index b656af46d..43a0804bd 100644 --- a/store/adapters/rdbms/drivers/sqlite/connect.go +++ b/store/adapters/rdbms/drivers/sqlite/connect.go @@ -34,7 +34,7 @@ var ( customDriver = &sqlite3.SQLiteDriver{ ConnectHook: func(conn *sqlite3.SQLiteConn) (err error) { // register regexp function and use Go's regexp fn - if err = conn.RegisterFunc("regexp", regexp.MatchString, true); err != nil { + if err = conn.RegisterFunc("regexp", fnRegExp, true); err != nil { return } @@ -47,6 +47,11 @@ var ( } ) +func fnRegExp(pattern string, s string) (matched bool, err error) { + matched, err = regexp.MatchString(pattern, s) + return +} + func init() { // register alter driver sql.Register(altSchema, customDriver) diff --git a/store/adapters/rdbms/drivers/sqlite/dialect.go b/store/adapters/rdbms/drivers/sqlite/dialect.go index 144dca1c4..f0a08a183 100644 --- a/store/adapters/rdbms/drivers/sqlite/dialect.go +++ b/store/adapters/rdbms/drivers/sqlite/dialect.go @@ -127,11 +127,7 @@ func (sqliteDialect) AttributeCast(attr *dal.Attribute, val exp.Expression) (exp c = exp.NewSQLFunctionExpression("strftime", "%Y-%m-%d", val) case *dal.TypeNumber: - ce := exp.NewCaseExpression(). - When(drivers.RegexpLike(drivers.CheckNumber, val), val). - Else(drivers.LiteralNULL) - - c = exp.NewCastExpression(ce, "NUMERIC") + c = exp.NewCastExpression(val, "NUMERIC") default: return drivers.AttributeCast(attr, val)