From cdacfe0648585b5f8b7a2fa32af67239f66aa12c Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Sat, 5 Nov 2022 11:52:01 +0100 Subject: [PATCH] Refactor RDBMS drivers, add json-array-contains A few changes to dialect interface, functions now return exp.Expression instead of more specific interfaces. Implement JSON array-contains functionality. This allows us to re-implement old behaviour where "'value' = multiValField" query inspected all values inside multi value field. --- store/adapters/rdbms/cursor.go | 8 +- store/adapters/rdbms/dal/iterator.go | 7 +- store/adapters/rdbms/dal/model.go | 18 +- store/adapters/rdbms/drivers/cast.go | 30 +-- store/adapters/rdbms/drivers/dialect.go | 47 ++++- store/adapters/rdbms/drivers/json.go | 108 ----------- store/adapters/rdbms/drivers/mysql/dialect.go | 76 ++++---- store/adapters/rdbms/drivers/mysql/json.go | 95 +++++++++ .../mysql/{dialect_test.go => json_test.go} | 0 .../rdbms/drivers/postgres/dialect.go | 45 +++-- store/adapters/rdbms/drivers/postgres/json.go | 42 ++++ .../rdbms/drivers/{ => postgres}/json_test.go | 70 +++---- .../adapters/rdbms/drivers/sqlite/connect.go | 8 +- .../adapters/rdbms/drivers/sqlite/dialect.go | 31 ++- store/adapters/rdbms/drivers/sqlite/json.go | 105 ++++++++++ .../rdbms/drivers/sqlite/json_test.go | 89 +++++++++ store/adapters/rdbms/drivers/table.go | 6 +- .../rdbms/drivers/tests/dialect_test.go | 182 +++++++++++++----- store/adapters/rdbms/filter.go | 13 +- 19 files changed, 680 insertions(+), 300 deletions(-) delete mode 100644 store/adapters/rdbms/drivers/json.go create mode 100644 store/adapters/rdbms/drivers/mysql/json.go rename store/adapters/rdbms/drivers/mysql/{dialect_test.go => json_test.go} (100%) create mode 100644 store/adapters/rdbms/drivers/postgres/json.go rename store/adapters/rdbms/drivers/{ => postgres}/json_test.go (55%) create mode 100644 store/adapters/rdbms/drivers/sqlite/json.go create mode 100644 store/adapters/rdbms/drivers/sqlite/json_test.go diff --git a/store/adapters/rdbms/cursor.go b/store/adapters/rdbms/cursor.go index 00f2089ea..c1da35a14 100644 --- a/store/adapters/rdbms/cursor.go +++ b/store/adapters/rdbms/cursor.go @@ -209,8 +209,8 @@ func (c *cursorCondition) sql() (cnd string, err error) { // check (on app-side) if value is nil to replace "? IS (NOT) NULL" check with TRUE/FALSE constants. func CursorExpression( cur *filter.PagingCursor, - identLookup func(i string) (exp.LiteralExpression, error), - castFn func(i string, val any) (exp.LiteralExpression, error), + identLookup func(i string) (exp.Expression, error), + castFn func(i string, val any) (exp.Expression, error), ) (e exp.Expression, err error) { var ( cc = cur.Keys() @@ -218,7 +218,7 @@ func CursorExpression( value any - ident exp.LiteralExpression + ident exp.Expression ltOp = map[bool]exp.BooleanOperation{ true: exp.LtOp, @@ -244,7 +244,7 @@ func CursorExpression( return false } - isValueNull = func(i int, neg bool) exp.LiteralExpression { + isValueNull = func(i int, neg bool) exp.Expression { if (nilCheck(vv[i]) && !neg) || (!nilCheck(vv[i]) && neg) { return exp.NewLiteralExpression("TRUE") } diff --git a/store/adapters/rdbms/dal/iterator.go b/store/adapters/rdbms/dal/iterator.go index 94f340787..0a05f2cd9 100644 --- a/store/adapters/rdbms/dal/iterator.go +++ b/store/adapters/rdbms/dal/iterator.go @@ -134,11 +134,8 @@ func (i *iterator) fetch(ctx context.Context) (rows *sql.Rows, err error) { // @todo this needs to work with embedded attributes (non physical columns) as well! tmp, err = rdbms.CursorExpression( cur, - func(ident string) (exp.LiteralExpression, error) { return i.src.table.AttributeExpression(ident) }, - func(ident string, val any) (exp.LiteralExpression, error) { - // @fixme vvv - // attr := i.dst.model.Attributes.FindByStoreIdent(ident) - + func(ident string) (exp.Expression, error) { return i.src.table.AttributeExpression(ident) }, + func(ident string, val any) (exp.Expression, error) { attr := i.dst.model.Attributes.FindByIdent(ident) if attr == nil { panic("unknown attribute " + ident + " used in cursor expression cast callback") diff --git a/store/adapters/rdbms/dal/model.go b/store/adapters/rdbms/dal/model.go index 27d6679a9..673ce0158 100644 --- a/store/adapters/rdbms/dal/model.go +++ b/store/adapters/rdbms/dal/model.go @@ -401,14 +401,14 @@ func (d *model) searchSql(f filter.Filter) *goqu.SelectDataset { // continue // } - var attrExpr exp.LiteralExpression + var attrExpr exp.Expression attrExpr, err = d.table.AttributeExpression(attr.Ident) if err != nil { return base.SetError(err) } if len(vv) > 0 { - cnd = append(cnd, attrExpr.In(vv...)) + cnd = append(cnd, exp.NewBooleanExpression(exp.InOp, attrExpr, vv)) } } @@ -434,7 +434,7 @@ func (d *model) searchSql(f filter.Filter) *goqu.SelectDataset { continue } - var attrExpr exp.LiteralExpression + var attrExpr exp.Expression attrExpr, err = d.table.AttributeExpression(attr.Ident) if err != nil { return base.SetError(err) @@ -443,11 +443,11 @@ func (d *model) searchSql(f filter.Filter) *goqu.SelectDataset { switch state { case filter.StateExclusive: // only not-null values - cnd = append(cnd, attrExpr.IsNotNull()) + cnd = append(cnd, exp.NewBooleanExpression(exp.IsNotOp, attrExpr, nil)) case filter.StateExcluded: // exclude all non-null values - cnd = append(cnd, attrExpr.IsNull()) + cnd = append(cnd, exp.NewBooleanExpression(exp.IsOp, attrExpr, nil)) } } @@ -458,20 +458,20 @@ func (d *model) searchSql(f filter.Filter) *goqu.SelectDataset { } var ( - metaKeyExpr exp.LiteralExpression + metaKeyExpr exp.Expression metaAttrIdent = exp.NewIdentifierExpression("", d.model.Ident, attr.Ident) ) for mKey, mVal := range f.MetaConstraints() { - metaKeyExpr, err = d.dialect.DeepIdentJSON(metaAttrIdent, mKey) + metaKeyExpr, err = d.dialect.JsonExtractUnquote(metaAttrIdent, mKey) if err != nil { return base.SetError(err) } if reflect2.IsNil(mVal) { - cnd = append(cnd, metaKeyExpr.IsNotNull()) + cnd = append(cnd, exp.NewBooleanExpression(exp.IsNotOp, metaKeyExpr, nil)) } else { - cnd = append(cnd, metaKeyExpr.Eq(mVal)) + cnd = append(cnd, exp.NewBooleanExpression(exp.EqOp, metaKeyExpr, mVal)) } } } diff --git a/store/adapters/rdbms/drivers/cast.go b/store/adapters/rdbms/drivers/cast.go index de50df3be..ac1122f41 100644 --- a/store/adapters/rdbms/drivers/cast.go +++ b/store/adapters/rdbms/drivers/cast.go @@ -16,7 +16,18 @@ var ( LiteralTRUE = exp.NewLiteralExpression(`TRUE`) ) -func AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func RegexpLike(format, val exp.Expression) exp.BooleanExpression { + return exp.NewBooleanExpression(exp.RegexpLikeOp, val, format) +} + +func BooleanCheck(val exp.Expression) exp.Expression { + return exp.NewCaseExpression(). + When(exp.NewBooleanExpression(exp.InOp, val, []any{LiteralTRUE, exp.NewLiteralExpression(`'true'`)}), LiteralTRUE). + When(exp.NewBooleanExpression(exp.InOp, val, []any{LiteralFALSE, exp.NewLiteralExpression(`'false'`)}), LiteralFALSE). + Else(LiteralNULL) +} + +func AttributeCast(attr *dal.Attribute, val exp.Expression) (exp.Expression, error) { var ( c exp.CastExpression ) @@ -24,46 +35,41 @@ func AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralE switch attr.Type.(type) { case *dal.TypeID, *dal.TypeRef: ce := exp.NewCaseExpression(). - When(val.RegexpLike(CheckID), val). + When(RegexpLike(CheckID, val), val). Else(LiteralNULL) c = exp.NewCastExpression(ce, "BIGINT") case *dal.TypeNumber: ce := exp.NewCaseExpression(). - When(val.RegexpLike(CheckNumber), val). + When(RegexpLike(CheckNumber, val), val). Else(LiteralNULL) c = exp.NewCastExpression(ce, "NUMERIC") case *dal.TypeTimestamp: ce := exp.NewCaseExpression(). - When(val.RegexpLike(CheckFullISO8061), val). + When(RegexpLike(CheckFullISO8061, val), val). Else(LiteralNULL) c = exp.NewCastExpression(ce, "TIMESTAMPTZ") case *dal.TypeDate: ce := exp.NewCaseExpression(). - When(val.RegexpLike(CheckDateISO8061), val). + When(RegexpLike(CheckDateISO8061, val), val). Else(LiteralNULL) c = exp.NewCastExpression(ce, "DATE") case *dal.TypeTime: ce := exp.NewCaseExpression(). - When(val.RegexpLike(CheckTimeISO8061), val). + When(RegexpLike(CheckTimeISO8061, val), val). Else(LiteralNULL) c = exp.NewCastExpression(ce, "TIMETZ") case *dal.TypeBoolean: - ce := exp.NewCaseExpression(). - When(val.In(LiteralTRUE, exp.NewLiteralExpression(`'true'`)), LiteralTRUE). - When(val.In(LiteralFALSE, exp.NewLiteralExpression(`'false'`)), LiteralFALSE). - Else(LiteralNULL) - - c = exp.NewCastExpression(ce, "BOOLEAN") + c = exp.NewCastExpression(BooleanCheck(val), "BOOLEAN") default: return val, nil diff --git a/store/adapters/rdbms/drivers/dialect.go b/store/adapters/rdbms/drivers/dialect.go index 50089d18f..7341aced0 100644 --- a/store/adapters/rdbms/drivers/dialect.go +++ b/store/adapters/rdbms/drivers/dialect.go @@ -1,6 +1,7 @@ package drivers import ( + "fmt" "github.com/cortezaproject/corteza-server/pkg/dal" "github.com/cortezaproject/corteza-server/pkg/ql" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" @@ -13,19 +14,25 @@ type ( // GOQU returns goqu's dialect wrapper struct GOQU() goqu.DialectWrapper - // DeepIdentJSON returns expression that allows us (read) access to a particular - // value inside JSON document: + // JsonExtract returns expression that returns a value from inside JSON document // - // DeepIdentJSON(exp.ParseExpression("some_column"), "a", "b") - // should result in something like: - // "some_column"->'a'->>'b' - // (depending on what is supported in the underlying database) - DeepIdentJSON(exp.IdentifierExpression, ...any) (exp.LiteralExpression, error) + // Use this when you want use JSON encoded value + JsonExtract(exp.Expression, ...any) (exp.Expression, error) + + // JsonExtractUnquote returns expression that returns a value from inside JSON document: + // + // Use this when you want to use unencoded value! + JsonExtractUnquote(exp.Expression, ...any) (exp.Expression, error) + + // JsonArrayContains generates expression JSON array containment check expression + // + // Literal values need to be JSON docs! + JsonArrayContains(needle, haystack exp.Expression) (exp.Expression, error) // AttributeCast prepares complex SQL expression that verifies // arbitrary string value in the db and casts it to b used in // comparison or soring expression - AttributeCast(*dal.Attribute, exp.LiteralExpression) (exp.LiteralExpression, error) + AttributeCast(*dal.Attribute, exp.Expression) (exp.Expression, error) // TableCodec returns table codec (encodes & decodes data to/from db table) TableCodec(*dal.Model) TableCodec @@ -48,3 +55,27 @@ type ( OrderedExpression(exp.Expression, exp.SortDirection, exp.NullSortType) exp.OrderedExpression } ) + +func init() { + goqu.SetDefaultPrepared(true) +} + +func IndexFieldModifiers(attr *dal.Attribute, quoteIdent func(i string) string, mm ...dal.IndexFieldModifier) (string, error) { + var ( + modifier string + out = quoteIdent(attr.StoreIdent()) + ) + + for _, m := range mm { + switch m { + case dal.IndexFieldModifierLower: + modifier = "LOWER" + default: + return "", fmt.Errorf("unknown index field modifier: %s", m) + } + + out = fmt.Sprintf("%s(%s)", modifier, out) + } + + return out, nil +} diff --git a/store/adapters/rdbms/drivers/json.go b/store/adapters/rdbms/drivers/json.go deleted file mode 100644 index 4bd4630a3..000000000 --- a/store/adapters/rdbms/drivers/json.go +++ /dev/null @@ -1,108 +0,0 @@ -package drivers - -// dialect.go -// -// Generic SQL functions used by majority of RDBMS drivers - -import ( - "fmt" - "github.com/cortezaproject/corteza-server/pkg/dal" - "strconv" - "strings" - - "github.com/doug-martin/goqu/v9" - "github.com/doug-martin/goqu/v9/exp" -) - -func init() { - goqu.SetDefaultPrepared(true) -} - -// DeepIdentJSON constructs expression with chain of JSON operators -// that point value inside JSON document -// -// Supported in databases: -// -// PostgreSQL -// https://www.postgresql.org/docs/9.3/functions-json.html -// -// MySQL -// https://dev.mysql.com/doc/expman/5.7/en/json-function-experence.html -// -// SQLite -// https://www.sqlite.org/json1.html#jptr -func DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) exp.LiteralExpression { - var ( - sql strings.Builder - last = len(pp) - 1 - ) - - sql.WriteString("?") - - for i, p := range pp { - sql.WriteString("-") - - sql.WriteString(">") - if i == last { - sql.WriteString(">") - } - - switch path := p.(type) { - case string: - sql.WriteString("'") - sql.WriteString(strings.ReplaceAll(path, "'", `\'`)) - sql.WriteString("'") - case int: - sql.WriteString(strconv.Itoa(path)) - default: - panic("invalid type") - } - } - - return exp.NewLiteralExpression(sql.String(), ident) -} - -// JsonPath constructs json-path string from the slice of path parts. -// -func JsonPath(pp ...any) string { - var ( - path strings.Builder - ) - - path.WriteString("$") - for i := range pp { - switch part := pp[i].(type) { - case string: - path.WriteString(".") - path.WriteString(part) - case int: - path.WriteString("[") - path.WriteString(strconv.Itoa(i)) - path.WriteString("]") - default: - panic(fmt.Errorf("JsonPath expect string or int, got %T", i)) - } - } - - return path.String() -} - -func IndexFieldModifiers(attr *dal.Attribute, quoteIdent func(i string) string, mm ...dal.IndexFieldModifier) (string, error) { - var ( - modifier string - out = quoteIdent(attr.StoreIdent()) - ) - - for _, m := range mm { - switch m { - case dal.IndexFieldModifierLower: - modifier = "LOWER" - default: - return "", fmt.Errorf("unknown index field modifier: %s", m) - } - - out = fmt.Sprintf("%s(%s)", modifier, out) - } - - return out, nil -} diff --git a/store/adapters/rdbms/drivers/mysql/dialect.go b/store/adapters/rdbms/drivers/mysql/dialect.go index fe8f7231b..22c4641bd 100644 --- a/store/adapters/rdbms/drivers/mysql/dialect.go +++ b/store/adapters/rdbms/drivers/mysql/dialect.go @@ -2,9 +2,6 @@ package mysql import ( "fmt" - "strconv" - "strings" - "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ql" @@ -38,8 +35,35 @@ func (d mysqlDialect) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexFi return drivers.IndexFieldModifiers(attr, d.QuoteIdent, mm...) } -func (mysqlDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { - return JSONPath(ident, pp...) +func (d mysqlDialect) JsonExtract(jsonDoc exp.Expression, pp ...any) (path exp.Expression, err error) { + if path, err = jsonPathExpr(pp...); err != nil { + return + } else { + return exp.NewSQLFunctionExpression("JSON_EXTRACT", jsonDoc, path), nil + } +} + +func (d mysqlDialect) JsonExtractUnquote(jsonDoc exp.Expression, pp ...any) (_ exp.Expression, err error) { + if jsonDoc, err = d.JsonExtract(jsonDoc, pp...); err != nil { + return + } else { + return exp.NewSQLFunctionExpression("JSON_UNQUOTE", jsonDoc), nil + } +} + +// JsonArrayContains prepares MySQL compatible comparison of value (or ident) and JSON array +// +// # literal value = multi-value field / plain +// # multi-value field = single-value field / plain +// JSON_CONTAINS(v, JSON_EXTRACT(needle, '$.f3'), '$.f2') +// +// # single-value field = multi-value field / plain +// # multi-value field = single-value field / plain +// JSON_CONTAINS(v, '"needle"', '$.f2') +// +// This approach is not optimal, but it is the only way to make it work +func (d mysqlDialect) JsonArrayContains(needle, haystack exp.Expression) (_ exp.Expression, err error) { + return exp.NewSQLFunctionExpression("JSON_CONTAINS", haystack, needle), nil } func (d mysqlDialect) TableCodec(m *dal.Model) drivers.TableCodec { @@ -68,7 +92,7 @@ func (d mysqlDialect) TypeWrap(dt dal.Type) drivers.Type { // AttributeCast for mySQL // // https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast -func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.Expression) (exp.Expression, error) { var ( c exp.CastExpression ) @@ -77,36 +101,31 @@ func (mysqlDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression case *dal.TypeNumber: ce := exp.NewCaseExpression(). - When(val.RegexpLike(drivers.CheckNumber), val). + When(drivers.RegexpLike(drivers.CheckNumber, val), val). Else(drivers.LiteralNULL) c = exp.NewCastExpression(ce, "DECIMAL(65,10)") case *dal.TypeTimestamp: ce := exp.NewCaseExpression(). - When(val.RegexpLike(drivers.CheckFullISO8061), val). + When(drivers.RegexpLike(drivers.CheckFullISO8061, val), val). Else(drivers.LiteralNULL) c = exp.NewCastExpression(ce, "DATETIME") case *dal.TypeBoolean: - ce := exp.NewCaseExpression(). - When(val.In(drivers.LiteralTRUE, exp.NewLiteralExpression(`'true'`)), drivers.LiteralTRUE). - When(val.In(drivers.LiteralFALSE, exp.NewLiteralExpression(`'false'`)), drivers.LiteralFALSE). - Else(drivers.LiteralNULL) - - c = exp.NewCastExpression(ce, "SIGNED") + c = exp.NewCastExpression(drivers.BooleanCheck(val), "SIGNED") case *dal.TypeID, *dal.TypeRef: ce := exp.NewCaseExpression(). - When(val.RegexpLike(drivers.CheckID), val). + When(drivers.RegexpLike(drivers.CheckID, val), val). Else(drivers.LiteralNULL) c = exp.NewCastExpression(ce, "UNSIGNED") case *dal.TypeTime: ce := exp.NewCaseExpression(). - When(val.RegexpLike(drivers.CheckTimeISO8061), val). + When(drivers.RegexpLike(drivers.CheckTimeISO8061, val), val). Else(drivers.LiteralNULL) c = exp.NewCastExpression(ce, "TIME") @@ -123,31 +142,6 @@ func (mysqlDialect) ExprHandler(n *ql.ASTNode, args ...exp.Expression) (exp.Expr return ql.DefaultRefHandler(n, args...) } -func JSONPath(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { - var ( - sql strings.Builder - ) - - sql.WriteString(`?->>'$`) - - for _, p := range pp { - switch path := p.(type) { - case string: - sql.WriteString(".") - sql.WriteString(strings.ReplaceAll(path, "'", `\'`)) - case int: - sql.WriteString("[") - sql.WriteString(strconv.Itoa(path)) - sql.WriteString("]") - default: - return nil, fmt.Errorf("unexpected path part (%q) type: %T", p, p) - } - } - - sql.WriteString(`'`) - return exp.NewLiteralExpression(sql.String(), ident), nil -} - func (mysqlDialect) AttributeToColumn(attr *dal.Attribute) (col *ddl.Column, err error) { col = &ddl.Column{ Ident: attr.StoreIdent(), diff --git a/store/adapters/rdbms/drivers/mysql/json.go b/store/adapters/rdbms/drivers/mysql/json.go new file mode 100644 index 000000000..66ce43421 --- /dev/null +++ b/store/adapters/rdbms/drivers/mysql/json.go @@ -0,0 +1,95 @@ +package mysql + +import ( + "fmt" + "github.com/doug-martin/goqu/v9/exp" + "strconv" + "strings" +) + +func JsonPath(asJSON bool, jsonDoc exp.Expression, pp ...any) (_ exp.LiteralExpression, err error) { + var ( + sql strings.Builder + path string + ) + + if path, err = jsonPath(pp...); err != nil { + return nil, err + } + + sql.WriteString(`?->'`) + if !asJSON { + // interested in un-encoded value + sql.WriteString(`>'`) + } + sql.WriteString(path) + sql.WriteString(`'`) + + return exp.NewLiteralExpression(sql.String(), jsonDoc), nil +} + +func jsonPathExpr(pp ...any) (exp.LiteralExpression, error) { + if path, err := jsonPath(pp...); err != nil { + return nil, err + } else { + return exp.NewLiteralExpression("'" + path + "'"), nil + } +} + +func jsonPath(pp ...any) (string, error) { + var ( + sql strings.Builder + ) + + sql.WriteString(`$`) + + for _, p := range pp { + switch path := p.(type) { + case string: + sql.WriteString(".") + sql.WriteString(strings.ReplaceAll(path, "'", `\'`)) + case int: + sql.WriteString("[") + sql.WriteString(strconv.Itoa(path)) + sql.WriteString("]") + default: + return "", fmt.Errorf("unexpected path part (%q) type: %T", p, p) + } + } + + return sql.String(), nil +} + +// JSONArrayContains prepares MySQL compatible comparison of value and JSON array +// +// # literal value = multi-value field / plain +// # multi-value field = single-value field / plain +// JSON_CONTAINS(v, JSON_EXTRACT(V, '$.f3'), '$.f2') +// +// # single-value field = multi-value field / plain +// # multi-value field = single-value field / plain +// JSON_CONTAINS(v, '"aaa"', '$.f2') +// +// This approach is not optimal, but it is the only way to make it work +func JSONArrayContains(left exp.Expression, ident exp.IdentifierExpression, pp ...any) (jc exp.Expression, err error) { + var ( + pathAux string + path exp.Expression + ) + + // this is the least painful way how to encode + // an unknown value as JSON + left = exp.NewSQLFunctionExpression( + "JSON_EXTRACT", + exp.NewSQLFunctionExpression("JSON_ARRAY", left), + exp.NewLiteralExpression(`'$[0]'`), + ) + + if pathAux, err = jsonPath(pp...); err != nil { + return nil, err + } else { + path = exp.NewLiteralExpression(`'` + pathAux + `'`) + } + + return exp.NewSQLFunctionExpression("JSON_CONTAINS", ident, left, path), nil +} diff --git a/store/adapters/rdbms/drivers/mysql/dialect_test.go b/store/adapters/rdbms/drivers/mysql/json_test.go similarity index 100% rename from store/adapters/rdbms/drivers/mysql/dialect_test.go rename to store/adapters/rdbms/drivers/mysql/json_test.go diff --git a/store/adapters/rdbms/drivers/postgres/dialect.go b/store/adapters/rdbms/drivers/postgres/dialect.go index ff93ab527..a3988e7ae 100644 --- a/store/adapters/rdbms/drivers/postgres/dialect.go +++ b/store/adapters/rdbms/drivers/postgres/dialect.go @@ -35,8 +35,23 @@ func (d postgresDialect) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.Inde return drivers.IndexFieldModifiers(attr, d.QuoteIdent, mm...) } -func (postgresDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { - return drivers.DeepIdentJSON(ident, pp...), nil +func (d postgresDialect) JsonExtract(ident exp.Expression, pp ...any) (exp.Expression, error) { + return DeepIdentJSON(true, ident, pp...), nil +} + +func (d postgresDialect) JsonExtractUnquote(ident exp.Expression, pp ...any) (exp.Expression, error) { + return DeepIdentJSON(false, ident, pp...), nil +} + +// JsonArrayContains prepares postgresql compatible comparison of value and JSON array +// +// literal value = multi-value field / plain +// 'value' <@ (v->'f0')::JSONB +// +// single-value field = multi-value field / plain +// v->'f1'->0 <@ (v->'f0')::JSONB +func (d postgresDialect) JsonArrayContains(needle, haystack exp.Expression) (exp.Expression, error) { + return exp.NewLiteralExpression("(?)::JSONB <@ (?)::JSONB", needle, haystack), nil } func (d postgresDialect) TableCodec(m *dal.Model) drivers.TableCodec { @@ -54,30 +69,24 @@ func (d postgresDialect) TypeWrap(dt dal.Type) drivers.Type { return drivers.TypeWrap(dt) } -func (postgresDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { - var ( - c exp.CastExpression - ) - +func (postgresDialect) AttributeCast(attr *dal.Attribute, val exp.Expression) (expr exp.Expression, err error) { switch attr.Type.(type) { - case *dal.TypeBoolean: - // we need to be strictly dealing with strings here! - // 1) postgresql's JSON op ->> (last one) returns any JSON value as string - // so booleans are cast to 'true' & 'false' - // 2) postgresql will complain about true == 'true' expressions - ce := exp.NewCaseExpression(). - When(val.In(exp.NewLiteralExpression(`'true'`)), drivers.LiteralTRUE). - When(val.In(exp.NewLiteralExpression(`'false'`)), drivers.LiteralFALSE). - Else(drivers.LiteralNULL) + case *dal.TypeText: + expr = exp.NewCastExpression(val, "TEXT") - c = exp.NewCastExpression(ce, "BOOLEAN") + case *dal.TypeBoolean: + // convert to text first + expr = exp.NewCastExpression(val, "TEXT") + + // compare to text representation of true + expr = exp.NewBooleanExpression(exp.EqOp, expr, exp.NewLiteralExpression(`true::TEXT`)) default: return drivers.AttributeCast(attr, val) } - return exp.NewLiteralExpression("?", c), nil + return } func (postgresDialect) AttributeToColumn(attr *dal.Attribute) (col *ddl.Column, err error) { diff --git a/store/adapters/rdbms/drivers/postgres/json.go b/store/adapters/rdbms/drivers/postgres/json.go new file mode 100644 index 000000000..31fb9898e --- /dev/null +++ b/store/adapters/rdbms/drivers/postgres/json.go @@ -0,0 +1,42 @@ +package postgres + +import ( + "github.com/doug-martin/goqu/v9/exp" + "strconv" + "strings" +) + +// DeepIdentJSON constructs expression with chain of JSON operators +// that point value inside JSON document +// +// https://www.postgresql.org/docs/9.3/functions-json.html +func DeepIdentJSON(asJSON bool, jsonDoc exp.Expression, pp ...any) exp.LiteralExpression { + var ( + sql strings.Builder + last = len(pp) - 1 + ) + + sql.WriteString("?") + + for i, p := range pp { + sql.WriteString("-") + + sql.WriteString(">") + if i == last && !asJSON { + sql.WriteString(">") + } + + switch path := p.(type) { + case string: + sql.WriteString("'") + sql.WriteString(strings.ReplaceAll(path, "'", `\'`)) + sql.WriteString("'") + case int: + sql.WriteString(strconv.Itoa(path)) + default: + panic("invalid type") + } + } + + return exp.NewLiteralExpression(sql.String(), jsonDoc) +} diff --git a/store/adapters/rdbms/drivers/json_test.go b/store/adapters/rdbms/drivers/postgres/json_test.go similarity index 55% rename from store/adapters/rdbms/drivers/json_test.go rename to store/adapters/rdbms/drivers/postgres/json_test.go index c1160a522..a88c66b9e 100644 --- a/store/adapters/rdbms/drivers/json_test.go +++ b/store/adapters/rdbms/drivers/postgres/json_test.go @@ -1,4 +1,4 @@ -package drivers +package postgres import ( "testing" @@ -17,9 +17,10 @@ func Test_DeepIdentJSON(t *testing.T) { post = ` FROM "test"` cc = []struct { - input []interface{} - sql string - args []interface{} + input []interface{} + sql string + asJSON bool + args []interface{} }{ { input: []interface{}{"one"}, @@ -41,6 +42,35 @@ func Test_DeepIdentJSON(t *testing.T) { sql: `"one"->'two'->>3`, args: []interface{}{}, }, + { + input: []interface{}{"one"}, + asJSON: true, + sql: `"one"`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", "two"}, + asJSON: true, + sql: `"one"->'two'`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", 2, "three"}, + asJSON: true, + sql: `"one"->2->'three'`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", "two", 3}, + asJSON: true, + sql: `"one"->'two'->3`, + args: []interface{}{}, + }, + } + + conv = func(asJSON bool, pp ...any) exp.SQLExpression { + return goqu.Dialect("postgres"). + Select(DeepIdentJSON(asJSON, exp.ParseIdentifier(pp[0].(string)), pp[1:]...)).From("test") } ) @@ -50,40 +80,10 @@ func Test_DeepIdentJSON(t *testing.T) { r = require.New(t) ) - sql, args, err := goqu.Dialect("postgres").Select(DeepIdentJSON(exp.ParseIdentifier(c.input[0].(string)), c.input[1:]...)).From("test").ToSQL() + sql, args, err := conv(c.asJSON, c.input...).ToSQL() r.NoError(err) r.Equal(pre+c.sql+post, sql) r.Equal(c.args, args) }) } } - -// test deep ident expression generator -func Test_JsonPath(t *testing.T) { - - var ( - cc = []struct { - input []interface{} - path string - }{ - { - input: []interface{}{"two"}, - path: `$.two`, - }, - { - input: []interface{}{2, "three"}, - path: `$[2].three`, - }, - { - input: []interface{}{"two", 3}, - path: `$.two[3]`, - }, - } - ) - - for _, c := range cc { - t.Run(c.path, func(t *testing.T) { - require.Equal(t, c.path, JsonPath(c.input...)) - }) - } -} diff --git a/store/adapters/rdbms/drivers/sqlite/connect.go b/store/adapters/rdbms/drivers/sqlite/connect.go index 7ae761e03..b656af46d 100644 --- a/store/adapters/rdbms/drivers/sqlite/connect.go +++ b/store/adapters/rdbms/drivers/sqlite/connect.go @@ -38,6 +38,10 @@ var ( return } + if err = conn.RegisterFunc("json_array_contains", sqliteFuncJsonArrayContains, true); err != nil { + return + } + return }, } @@ -46,7 +50,7 @@ var ( func init() { // register alter driver sql.Register(altSchema, customDriver) - // register drbug driver + // register debug driver sql.Register(debugSchema, sqlmw.Driver(customDriver, instrumentation.Debug())) store.Register(Connect, SCHEMA, altSchema, debugSchema) @@ -86,7 +90,7 @@ func Connect(ctx context.Context, dsn string) (_ store.Storer, err error) { } func ConnectInMemory(ctx context.Context) (s store.Storer, err error) { - return Connect(ctx, SCHEMA+"://file::memory:?cache=shared&mode=memory") + return Connect(ctx, altSchema+"://file::memory:?cache=shared&mode=memory") } func ConnectInMemoryWithDebug(ctx context.Context) (s store.Storer, err error) { diff --git a/store/adapters/rdbms/drivers/sqlite/dialect.go b/store/adapters/rdbms/drivers/sqlite/dialect.go index cf1002964..3a1ea1793 100644 --- a/store/adapters/rdbms/drivers/sqlite/dialect.go +++ b/store/adapters/rdbms/drivers/sqlite/dialect.go @@ -10,7 +10,6 @@ import ( "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/dialect/sqlite3" "github.com/doug-martin/goqu/v9/exp" - "regexp" "strings" ) @@ -47,8 +46,29 @@ func (d sqliteDialect) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexF return drivers.IndexFieldModifiers(attr, d.QuoteIdent, mm...) } -func (sqliteDialect) DeepIdentJSON(ident exp.IdentifierExpression, pp ...any) (exp.LiteralExpression, error) { - return drivers.DeepIdentJSON(ident, pp...), nil +func (sqliteDialect) JsonExtract(ident exp.Expression, pp ...any) (exp.Expression, error) { + return DeepIdentJSON(true, ident, pp...), nil +} + +func (sqliteDialect) JsonExtractUnquote(ident exp.Expression, pp ...any) (exp.Expression, error) { + return DeepIdentJSON(false, ident, pp...), nil +} + +// JsonArrayContains prepares SQLite compatible comparison of value and JSON array +// +// # literal value = multi-value field / plain +// # multi-value field = single-value field / plain +// +// 'aaa' in (select value from json_each(v->'f2')) +// +// # single-value field = multi-value field / plain +// # multi-value field = single-value field / plain +// json_extract(v, '$.f3[0]') in (select value from json_each(v->'f2')); +// +// Unfortunately SQLite converts boolean values into 0 and 1 when decoding from +// JSON and we need a special handler for that. +func (sqliteDialect) JsonArrayContains(needle, haystack exp.Expression) (exp.Expression, error) { + return exp.NewLiteralExpression("JSON_ARRAY_CONTAINS(?, ?)", needle, haystack), nil } func (d sqliteDialect) TableCodec(m *dal.Model) drivers.TableCodec { @@ -71,7 +91,7 @@ func (d sqliteDialect) TypeWrap(dt dal.Type) drivers.Type { return drivers.TypeWrap(dt) } -func (sqliteDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpression) (exp.LiteralExpression, error) { +func (sqliteDialect) AttributeCast(attr *dal.Attribute, val exp.Expression) (exp.Expression, error) { var ( c exp.Expression ) @@ -99,9 +119,8 @@ func (sqliteDialect) AttributeCast(attr *dal.Attribute, val exp.LiteralExpressio c = exp.NewSQLFunctionExpression("strftime", "%Y-%m-%d", val) case *dal.TypeNumber: - match, _ := regexp.Match(drivers.CheckNumber.Literal(), []byte(val.Literal())) ce := exp.NewCaseExpression(). - When(val.In(match, val), val). + When(drivers.RegexpLike(drivers.CheckNumber, val), val). Else(drivers.LiteralNULL) c = exp.NewCastExpression(ce, "NUMERIC") diff --git a/store/adapters/rdbms/drivers/sqlite/json.go b/store/adapters/rdbms/drivers/sqlite/json.go new file mode 100644 index 000000000..a2f5fcc51 --- /dev/null +++ b/store/adapters/rdbms/drivers/sqlite/json.go @@ -0,0 +1,105 @@ +package sqlite + +import ( + "fmt" + "github.com/doug-martin/goqu/v9/exp" + "github.com/valyala/fastjson" + "strconv" + "strings" +) + +// DeepIdentJSON constructs expression with chain of JSON operators +// that point value inside JSON document +// +// https://www.postgresql.org/docs/9.3/functions-json.html +func DeepIdentJSON(asJSON bool, jsonDoc exp.Expression, pp ...any) exp.LiteralExpression { + var ( + sql strings.Builder + last = len(pp) - 1 + ) + + sql.WriteString("?") + + for i, p := range pp { + sql.WriteString("-") + + sql.WriteString(">") + if i == last && !asJSON { + sql.WriteString(">") + } + + switch path := p.(type) { + case string: + sql.WriteString("'") + sql.WriteString(strings.ReplaceAll(path, "'", `\'`)) + sql.WriteString("'") + case int: + sql.WriteString(strconv.Itoa(path)) + default: + panic("invalid type") + } + } + + return exp.NewLiteralExpression(sql.String(), jsonDoc) +} + +func jsonPath(pp ...any) (string, error) { + var ( + sql strings.Builder + ) + + sql.WriteString(`$`) + + for _, p := range pp { + switch path := p.(type) { + case string: + sql.WriteString(".") + sql.WriteString(strings.ReplaceAll(path, "'", `\'`)) + case int: + sql.WriteString("[") + sql.WriteString(strconv.Itoa(path)) + sql.WriteString("]") + default: + return "", fmt.Errorf("unexpected path part (%q) type: %T", p, p) + } + } + + return sql.String(), nil +} + +func sqliteFuncJsonArrayContains(needle, haystack []byte) (_ bool, err error) { + println("FOO") + var n, h, i *fastjson.Value + + if h, err = fastjson.ParseBytes(haystack); err != nil { + return + } + + if h.Type() != fastjson.TypeArray { + err = fmt.Errorf("haystack is %s, expecting array", h.Type()) + return + } + + if n, err = fastjson.ParseBytes(needle); err != nil { + return + } + + for _, i = range h.GetArray() { + if i.Type() != n.Type() { + continue + } + + switch i.Type() { + case fastjson.TypeFalse, fastjson.TypeTrue: + if i.GetBool() == n.GetBool() { + return true, nil + } + case fastjson.TypeNumber, fastjson.TypeString: + if i.String() == n.String() { + return true, nil + } + } + } + + return +} diff --git a/store/adapters/rdbms/drivers/sqlite/json_test.go b/store/adapters/rdbms/drivers/sqlite/json_test.go new file mode 100644 index 000000000..c4d76a83c --- /dev/null +++ b/store/adapters/rdbms/drivers/sqlite/json_test.go @@ -0,0 +1,89 @@ +package sqlite + +import ( + "testing" + + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" + "github.com/stretchr/testify/require" + + _ "github.com/doug-martin/goqu/v9/dialect/sqlite3" +) + +// test deep ident expression generator +func Test_DeepIdentJSON(t *testing.T) { + var ( + pre = `SELECT ` + post = ` FROM "test"` + + cc = []struct { + input []interface{} + sql string + asJSON bool + args []interface{} + }{ + { + input: []interface{}{"one"}, + sql: `"one"`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", "two"}, + sql: `"one"->>'two'`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", 2, "three"}, + sql: `"one"->2->>'three'`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", "two", 3}, + sql: `"one"->'two'->>3`, + args: []interface{}{}, + }, + { + input: []interface{}{"one"}, + asJSON: true, + sql: `"one"`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", "two"}, + asJSON: true, + sql: `"one"->'two'`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", 2, "three"}, + asJSON: true, + sql: `"one"->2->'three'`, + args: []interface{}{}, + }, + { + input: []interface{}{"one", "two", 3}, + asJSON: true, + sql: `"one"->'two'->3`, + args: []interface{}{}, + }, + } + + conv = func(asJSON bool, pp ...any) exp.SQLExpression { + return goqu.Dialect("postgres"). + Select(DeepIdentJSON(asJSON, exp.ParseIdentifier(pp[0].(string)), pp[1:]...)).From("test") + } + ) + + for _, c := range cc { + t.Run(c.sql, func(t *testing.T) { + var ( + r = require.New(t) + ) + + sql, args, err := conv(c.asJSON, c.input...).ToSQL() + r.NoError(err) + r.Equal(pre+c.sql+post, sql) + r.Equal(c.args, args) + }) + } +} diff --git a/store/adapters/rdbms/drivers/table.go b/store/adapters/rdbms/drivers/table.go index 60851d5ec..bbca59408 100644 --- a/store/adapters/rdbms/drivers/table.go +++ b/store/adapters/rdbms/drivers/table.go @@ -15,7 +15,7 @@ type ( MakeScanBuffer() []any Encode(r dal.ValueGetter) (_ []any, err error) Decode(buf []any, r dal.ValueSetter) (err error) - AttributeExpression(string) (exp.LiteralExpression, error) + AttributeExpression(string) (exp.Expression, error) } // GenericTableCodec is a generic implementation of TableCodec @@ -123,7 +123,7 @@ func (t *GenericTableCodec) Decode(buf []any, r dal.ValueSetter) (err error) { return } -func (t *GenericTableCodec) AttributeExpression(ident string) (exp.LiteralExpression, error) { +func (t *GenericTableCodec) AttributeExpression(ident string) (exp.Expression, error) { attr := t.model.Attributes.FindByIdent(ident) if attr == nil { @@ -137,7 +137,7 @@ func (t *GenericTableCodec) AttributeExpression(ident string) (exp.LiteralExpres case *dal.CodecRecordValueSetJSON: // using JSON to handle embedded values - lit, err := t.dialect.DeepIdentJSON(exp.NewIdentifierExpression("", t.model.Ident, s.Ident), attr.Ident, 0) + lit, err := t.dialect.JsonExtractUnquote(exp.NewIdentifierExpression("", t.model.Ident, s.Ident), attr.Ident, 0) if err != nil { return nil, err } diff --git a/store/adapters/rdbms/drivers/tests/dialect_test.go b/store/adapters/rdbms/drivers/tests/dialect_test.go index 9ea31a195..d9f2d4421 100644 --- a/store/adapters/rdbms/drivers/tests/dialect_test.go +++ b/store/adapters/rdbms/drivers/tests/dialect_test.go @@ -1,77 +1,169 @@ package tests import ( - "fmt" + "encoding/json" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/errors" + "github.com/cortezaproject/corteza-server/store/adapters/rdbms/ddl" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" "github.com/stretchr/testify/require" "testing" ) -func TestDeepIdentJSON(t *testing.T) { - const ( - tbl = "test_json_path_test" - ) - +func TestJSONOp(t *testing.T) { var ( + tbl = ddl.Table{ + Ident: "test_json_path_test", + Columns: []*ddl.Column{ + {Ident: "c", Type: &ddl.ColumnType{Name: "JSON"}}, + }, + Temporary: true, + } + req = require.New(t) - count = func(val string) int { + // test utility function + // counts how many rows have a value in the json path + count = func(req *require.Assertions, expr ...exp.Expression) int { var ( out = struct { Count int `db:"count"` }{} - - diJSON exp.Expression - err error ) - diJSON, err = conn.dialect.DeepIdentJSON( - exp.NewIdentifierExpression("", tbl, "c"), - "a", "b", "c", - ) - req.NoError(err) query := conn.dialect.GOQU(). Select(goqu.COUNT(goqu.Star()).As("count")). - From(tbl). - Where(exp.NewLiteralExpression("?", diJSON).Eq(val)) + From(tbl.Ident). + Where(expr...) - err = conn.store.QueryOne(ctx, query, &out) + err := conn.store.QueryOne(ctx, query, &out) req.NoError(err) return out.Count } + + // test utility function + // counts how many rows have a value in the json path + countDeepJSON = func(req *require.Assertions, val string) int { + col := exp.NewIdentifierExpression("", tbl.Ident, "c") + diJSON, err := conn.dialect.JsonExtractUnquote(col, "a", "b", "c") + req.NoError(err) + + return count(req, exp.NewLiteralExpression("?", diJSON).Eq(val)) + } + + a2e = func(attr *dal.Attribute) exp.Expression { + expr, err := conn.dialect.JsonExtract( + exp.NewIdentifierExpression("", tbl.Ident, attr.StoreIdent()), + attr.Ident, + ) + + req.NoError(err) + return expr + } + + // test utility function + // counts how many rows have a value in the json path + countContains = func(req *require.Assertions, attr *dal.Attribute, val any) int { + contains, err := conn.dialect.JsonArrayContains( + exp.NewLiteralExpression("?", val), + a2e(attr), + ) + req.NoError(err) + + return count(req, contains) + } + + // test utility function + insert = func(vv ...string) { + for _, val := range vv { + insert := conn.dialect.GOQU(). + Insert(tbl.Ident). + Cols("c"). + Vals(goqu.Vals{val}) + + req.NoError(conn.store.Exec(ctx, insert)) + } + } + + asJSON = func(val any) string { + enc, err := json.Marshal(val) + req.NoError(err) + return string(enc) + } + + jsonStore = &dal.CodecRecordValueSetJSON{Ident: "c"} + numAttr = &dal.Attribute{Ident: "n", Type: &dal.TypeNumber{}, Store: jsonStore} + textAttr = &dal.Attribute{Ident: "s", Type: &dal.TypeText{}, Store: jsonStore} + boolAttr = &dal.Attribute{Ident: "b", Type: &dal.TypeBoolean{}, Store: jsonStore} + + numCheck0Attr = &dal.Attribute{Ident: "nCheck0", Type: &dal.TypeNumber{}, Store: jsonStore} + textCheck0Attr = &dal.Attribute{Ident: "sCheck0", Type: &dal.TypeText{}, Store: jsonStore} + boolCheck0Attr = &dal.Attribute{Ident: "bCheck0", Type: &dal.TypeBoolean{}, Store: jsonStore} + + numCheck1Attr = &dal.Attribute{Ident: "nCheck1", Type: &dal.TypeNumber{}, Store: jsonStore} + textCheck1Attr = &dal.Attribute{Ident: "sCheck1", Type: &dal.TypeText{}, Store: jsonStore} + boolCheck1Attr = &dal.Attribute{Ident: "bCheck1", Type: &dal.TypeBoolean{}, Store: jsonStore} ) - err := makeTableWithJsonColumn(tbl) - if err != nil { - t.Fatalf("can not create table: %v", err) + { + // check if table exists, drop it and create a new, temporary one + + dd := conn.store.DataDefiner + + // utilize DDL and create a table with a json column + _, err := dd.TableLookup(ctx, tbl.Ident) + if errors.IsNotFound(err) { + err = nil + } else if err == nil { + err = dd.TableDrop(ctx, tbl.Ident) + } + + req.NoError(err) + req.NoError(dd.TableCreate(ctx, &tbl)) } - insert := conn.dialect.GOQU(). - Insert(tbl). - Cols("c"). - Vals([]any{`{"a": {"b": {"c": "match"}}}`}) + t.Run("deep ident json", func(t *testing.T) { + req = require.New(t) + req.NoError(conn.store.Exec(ctx, conn.dialect.GOQU().Truncate(tbl.Ident))) - req.NoError(conn.store.Exec(ctx, insert)) + insert(`{"a": {"b": {"c": "match"}}}`) - req.Equal(1, count("match")) - req.Equal(0, count("nope")) -} - -func makeTableWithJsonColumn(tbl string) (err error) { - if err = exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tbl)); err != nil { - return - } - - switch { - case conn.isSQLite, conn.isPostgres: - return exec(fmt.Sprintf(`CREATE TABLE %s (c JSONB)`, tbl)) - - case conn.isMySQL: - return exec(fmt.Sprintf(`CREATE TABLE %s (c TEXT)`, tbl)) - - default: - return fmt.Errorf("unsupported driver: %q", conn.config.DriverName) - } + req.Equal(1, countDeepJSON(req, "match")) + req.Equal(0, countDeepJSON(req, "nope")) + }) + + t.Run("json array contains", func(t *testing.T) { + req = require.New(t) + req.NoError(conn.store.Exec(ctx, conn.dialect.GOQU().Truncate(tbl.Ident))) + + insert(`{"n": [1,2,3], "b": [true], "s": ["foo", "bar"], "nCheck0": 0, "bCheck0": false, "sCheck0": "baz", "nCheck1": 1, "bCheck1": true, "sCheck1": "foo"}`) + + t.Log("Validating contains check with numeric value") + req.Equal(1, countContains(req, numAttr, asJSON(1))) + req.Equal(0, countContains(req, numAttr, asJSON(0))) + + t.Log("Validating contains check with string value") + req.Equal(1, countContains(req, textAttr, asJSON(`foo`))) + req.Equal(1, countContains(req, textAttr, asJSON(`bar`))) + req.Equal(0, countContains(req, textAttr, asJSON(`baz`))) + + t.Log("Validating contains check with boolean value") + req.Equal(1, countContains(req, boolAttr, asJSON(true))) + req.Equal(0, countContains(req, boolAttr, asJSON(false))) + + t.Log("Validating contains check with numeric field") + req.Equal(1, countContains(req, numAttr, a2e(numCheck1Attr))) + req.Equal(0, countContains(req, numAttr, a2e(numCheck0Attr))) + + t.Log("Validating contains check with string field") + req.Equal(1, countContains(req, textAttr, a2e(textCheck1Attr))) + req.Equal(1, countContains(req, textAttr, a2e(textCheck1Attr))) + req.Equal(0, countContains(req, textAttr, a2e(textCheck0Attr))) + + t.Log("Validating contains check with boolean field") + req.Equal(1, countContains(req, boolAttr, a2e(boolCheck1Attr))) + req.Equal(0, countContains(req, boolAttr, a2e(boolCheck0Attr))) + }) } diff --git a/store/adapters/rdbms/filter.go b/store/adapters/rdbms/filter.go index c6c4deb71..52a3c4560 100644 --- a/store/adapters/rdbms/filter.go +++ b/store/adapters/rdbms/filter.go @@ -313,14 +313,19 @@ func DefaultFilters() (f *extendedFilters) { if f.SubWorkflow != filter.StateInclusive { vattr := &dal.Attribute{Type: &dal.TypeBoolean{}} - litexp, _ := s.Dialect.DeepIdentJSON(goqu.C("meta"), "subWorkflow") - litexp, _ = s.Dialect.AttributeCast(vattr, litexp) + expr, _ := s.Dialect.JsonExtractUnquote(goqu.C("meta"), "subWorkflow") + expr, _ = s.Dialect.AttributeCast(vattr, expr) switch f.SubWorkflow { case filter.StateExcluded: - ee = append(ee, goqu.Or(litexp.IsFalse(), litexp.IsNull())) + ee = append(ee, goqu.Or( + exp.NewBooleanExpression(exp.EqOp, expr, false), + exp.NewBooleanExpression(exp.IsOp, expr, nil), + )) case filter.StateExclusive: - ee = append(ee, litexp.IsTrue()) + ee = append(ee, + exp.NewBooleanExpression(exp.EqOp, expr, true), + ) } }