From dcd3beeced00975b52548452cca448592f5cfdc6 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Wed, 11 Nov 2020 22:09:52 +0100 Subject: [PATCH] Implement value expressions on fields --- compose/service/record.go | 12 ++ compose/service/values/expr.go | 113 ++++++++++++++++ compose/service/values/expr_test.go | 165 +++++++++++++++++++++++ compose/service/values/sanitizer.go | 120 +++++++---------- compose/service/values/sanitizer_test.go | 6 +- compose/service/values/shared.go | 4 - compose/service/values/validator.go | 6 +- compose/types/module_field.go | 12 ++ compose/types/module_field_options.go | 34 +++++ compose/types/record.go | 17 +++ compose/types/record_value.go | 70 ++++++++++ 11 files changed, 476 insertions(+), 83 deletions(-) create mode 100644 compose/service/values/expr.go create mode 100644 compose/service/values/expr_test.go diff --git a/compose/service/record.go b/compose/service/record.go index 637804e6a..bf1719d03 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -760,6 +760,12 @@ func (svc record) procCreate(ctx context.Context, s store.Storer, invokerID uint return rve } + values.Expression(ctx, m, new, nil, rve) + + if !rve.IsValid() { + return rve + } + // Run validation of the updated records return svc.validator.Run(ctx, s, m, new) } @@ -846,6 +852,12 @@ func (svc record) procUpdate(ctx context.Context, s store.Storer, invokerID uint return rve } + values.Expression(ctx, m, upd, old, rve) + + if !rve.IsValid() { + return rve + } + // Run validation of the updated records return svc.validator.Run(ctx, s, m, upd) } diff --git a/compose/service/values/expr.go b/compose/service/values/expr.go new file mode 100644 index 000000000..5dcd4e4dc --- /dev/null +++ b/compose/service/values/expr.go @@ -0,0 +1,113 @@ +package values + +import ( + "context" + "fmt" + "github.com/PaesslerAG/gval" + "github.com/cortezaproject/corteza-server/compose/types" +) + +func makeInvalidExprErr(field *types.ModuleField, expr string, err error) types.RecordValueError { + return types.RecordValueError{ + Kind: "valueExpression", + Message: fmt.Sprintf("invalid expression %q: %v", expr, err.Error()), + Meta: map[string]interface{}{"field": field.Name}, + } +} +func makeExprEvalErr(field *types.ModuleField, expr string, err error) types.RecordValueError { + return types.RecordValueError{ + Kind: "valueExpression", + Message: fmt.Sprintf("failed to evaluate formula expression %q: %v", expr, err.Error()), + Meta: map[string]interface{}{"field": field.Name}, + } +} + +func makeValueExprIncompErr(field *types.ModuleField) types.RecordValueError { + return types.RecordValueError{ + Kind: "evaluatedValueIncompatible", + Message: "evaluated results incompatible", + Meta: map[string]interface{}{"field": field.Name}, + } +} + +func parser() gval.Language { + return gval.Full() +} + +// Expression evaluates expression in ModuleField.Expressions.Value and +// assigns results to the record on that field +func Expression(ctx context.Context, m *types.Module, r *types.Record, old *types.Record, rve *types.RecordValueErrorSet) { + var ( + exprParser = parser() + + scope = make(map[string]interface{}) + + reserved = map[string]bool{ + "new": true, + "old": true, + } + ) + + // base scope with field=value(s) from new record + scope = r.Values.Dict(m.Fields) + + // new record + scope["new"] = r.Dict(m) + + if old != nil { + // old values on record (before update) + // this will not be set for new records + scope["old"] = old.Dict(m) + } + + for _, f := range m.Fields { + if f.Expressions.Value == "" { + continue + } + + expr := f.Expressions.Value + + eval, err := exprParser.NewEvaluable(expr) + if err != nil { + rve.Push(makeInvalidExprErr(f, expr, err)) + return + } + + tmp, err := eval(ctx, scope) + if err != nil { + rve.Push(makeExprEvalErr(f, expr, err)) + return + } + + var strings []string + if values, isSlice := tmp.([]interface{}); isSlice { + if !f.Multi { + rve.Push(makeValueExprIncompErr(f)) + continue + } + + strings = make([]string, len(values)) + for i, value := range values { + strings[i] = sanitize(f, value) + } + } else { + if f.Multi { + rve.Push(makeValueExprIncompErr(f)) + continue + } + + strings = []string{sanitize(f, tmp)} + + } + + r.Values = r.Values.Replace(f.Name, strings...) + + if !reserved[f.Name] { + // make sure we do not overrider reserved fields + scope[f.Name] = tmp + } + + // Reset $new with updated data + scope["new"] = r.Dict(m) + } +} diff --git a/compose/service/values/expr_test.go b/compose/service/values/expr_test.go new file mode 100644 index 000000000..c585aadbb --- /dev/null +++ b/compose/service/values/expr_test.go @@ -0,0 +1,165 @@ +package values + +import ( + "context" + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/stretchr/testify/require" + "testing" +) + +func TestExpressions(t *testing.T) { + var ( + ctx = context.Background() + + // pairs = [, , ...] + makeModule = func(pairs ...string) *types.Module { + var ( + m = &types.Module{} + ) + + for i := 0; i < len(pairs); i += 3 { + f := &types.ModuleField{Name: pairs[i], Kind: pairs[i+1], Options: map[string]interface{}{}} + f.Expressions.Value = pairs[i+2] + m.Fields = append(m.Fields, f) + } + + return m + } + ) + + t.Run("empty", func(t *testing.T) { + var ( + req = require.New(t) + m = &types.Module{} + r = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + Expression(ctx, m, r, nil, rve) + req.True(rve.IsValid()) + }) + + t.Run("string", func(t *testing.T) { + var ( + req = require.New(t) + m = makeModule("f1", "String", `"abc"`) + r = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + Expression(ctx, m, r, nil, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("abc", r.Values.Get("f1", 0).Value) + }) + + t.Run("use fields", func(t *testing.T) { + var ( + req = require.New(t) + m = makeModule("f1", "String", `fname + " " + lname`) + r = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + m.Fields = append(m.Fields, &types.ModuleField{Name: "fname", Kind: "String"}) + m.Fields = append(m.Fields, &types.ModuleField{Name: "lname", Kind: "String"}) + r.Values = r.Values.Replace("fname", "Cor") + r.Values = r.Values.Replace("lname", "Teza") + + Expression(ctx, m, r, nil, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("Cor Teza", r.Values.Get("f1", 0).Value) + }) + + t.Run("math", func(t *testing.T) { + var ( + req = require.New(t) + m = makeModule("f1", "Number", `n1 * n2 * 1.511`) + r = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + m.Fields.FindByName("f1").Options.SetPrecision(1) + + m.Fields = append(m.Fields, &types.ModuleField{Name: "n1", Kind: "Number"}) + m.Fields = append(m.Fields, &types.ModuleField{Name: "n2", Kind: "Number"}) + r.Values = r.Values.Replace("n1", "10") + r.Values = r.Values.Replace("n2", "20") + + Expression(ctx, m, r, nil, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("302.2", r.Values.Get("f1", 0).Value) + }) + + t.Run("booleans", func(t *testing.T) { + var ( + m = makeModule("f1", "string", `b1 ? "yes" : "no"`) + r = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + m.Fields = append(m.Fields, &types.ModuleField{Name: "b1", Kind: "Bool"}) + + t.Run("true", func(t *testing.T) { + var req = require.New(t) + r.Values = r.Values.Replace("b1", "true") + + Expression(ctx, m, r, nil, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("yes", r.Values.Get("f1", 0).Value) + }) + + t.Run("false", func(t *testing.T) { + var req = require.New(t) + r.Values = r.Values.Replace("b1", "false") + + Expression(ctx, m, r, nil, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("no", r.Values.Get("f1", 0).Value) + }) + }) + + t.Run("old vs new", func(t *testing.T) { + var ( + req = require.New(t) + m = makeModule("f1", "String", `new.values.test == old.values.test ? "same":"different"`) + new = &types.Record{} + old = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + m.Fields = append(m.Fields, &types.ModuleField{Name: "test", Kind: "String"}) + + new.Values = new.Values.Replace("test", "a") + old.Values = old.Values.Replace("test", "a") + + Expression(ctx, m, new, old, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("same", new.Values.Get("f1", 0).Value) + }) + + t.Run("multi value expressions", func(t *testing.T) { + var ( + req = require.New(t) + m = makeModule("f1", "String", `[t1,t2,t3]`) + new = &types.Record{} + old = &types.Record{} + rve = &types.RecordValueErrorSet{} + ) + + m.Fields.FindByName("f1").Multi = true + + m.Fields = append(m.Fields, &types.ModuleField{Name: "t1", Kind: "String"}) + m.Fields = append(m.Fields, &types.ModuleField{Name: "t2", Kind: "String"}) + m.Fields = append(m.Fields, &types.ModuleField{Name: "t3", Kind: "String"}) + + new.Values = new.Values.Replace("t1", "a") + new.Values = new.Values.Replace("t2", "b") + new.Values = new.Values.Replace("t3", "c") + + Expression(ctx, m, new, old, rve) + req.Truef(rve.IsValid(), "%v", rve.Set) + req.Equal("a", new.Values.Get("f1", 0).Value) + req.Equal("b", new.Values.Get("f1", 1).Value) + req.Equal("c", new.Values.Get("f1", 2).Value) + }) +} diff --git a/compose/service/values/sanitizer.go b/compose/service/values/sanitizer.go index edb297ca5..8ad2ebff8 100644 --- a/compose/service/values/sanitizer.go +++ b/compose/service/values/sanitizer.go @@ -1,6 +1,7 @@ package values import ( + "fmt" "strconv" "strings" "time" @@ -86,11 +87,13 @@ func (s sanitizer) Run(m *types.Module, vv types.RecordValueSet) (out types.Reco // Per field type validators switch strings.ToLower(f.Kind) { case "bool": - v = s.sBool(v, f, m) + v.Value = sBool(v.Value) + case "datetime": - v = s.sDatetime(v, f, m) + v.Value = sDatetime(v.Value, f.Options.Bool("onlyDate"), f.Options.Bool("onlyTime")) + case "number": - v = s.sNumber(v, f, m) + v.Value = sNumber(v.Value, f.Options.Precision()) // Uncomment when they become relevant for sanitization //case "email": @@ -113,26 +116,34 @@ func (s sanitizer) Run(m *types.Module, vv types.RecordValueSet) (out types.Reco return } -func (sanitizer) sBool(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { - if truthy.MatchString(strings.ToLower(v.Value)) { - v.Value = strBoolTrue - } else { - v.Value = strBoolFalse +func sBool(v interface{}) string { + switch c := v.(type) { + case bool: + if c { + return strBoolTrue + } + + case string: + if truthy.MatchString(strings.ToLower(c)) { + return strBoolTrue + } } - return v + return strBoolFalse } -func (sanitizer) sDatetime(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { +func sDatetime(v interface{}, onlyDate, onlyTime bool) string { var ( // input format set inputFormats []string // output format internalFormat string + + datetime = fmt.Sprintf("%v", v) ) - if f.Options.Bool("onlyDate") { + if onlyDate { internalFormat = datetimeInternalFormatDate inputFormats = []string{ datetimeInternalFormatDate, @@ -141,7 +152,7 @@ func (sanitizer) sDatetime(v *types.RecordValue, f *types.ModuleField, m *types. "Mon, 02 Jan 2006", "2006/_1/_2", } - } else if f.Options.Bool("onlyTime") { + } else if onlyTime { internalFormat = datetimeIntenralFormatTime inputFormats = []string{ datetimeIntenralFormatTime, @@ -172,9 +183,9 @@ func (sanitizer) sDatetime(v *types.RecordValue, f *types.ModuleField, m *types. } // if string looks like a RFC 3330 (ISO 8601), see if we need to suffix it with Z - if isoDaty.MatchString(v.Value) && !hasTimezone.MatchString(v.Value) { + if isoDaty.MatchString(datetime) && !hasTimezone.MatchString(datetime) { // No timezone, add Z to satisfy parser - v.Value = v.Value + "Z" + datetime = datetime + "Z" // Simplifiy list of rules inputFormats = []string{time.RFC3339} @@ -182,76 +193,43 @@ func (sanitizer) sDatetime(v *types.RecordValue, f *types.ModuleField, m *types. } for _, format := range inputFormats { - parsed, err := time.Parse(format, v.Value) + parsed, err := time.Parse(format, datetime) if err == nil { - v.Value = parsed.UTC().Format(internalFormat) - return v + return parsed.UTC().Format(internalFormat) } } - v.Value = "" - return v + return "" } -// sNumber sanitizes -func (sanitizer) sNumber(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { - base, err := strconv.ParseFloat(v.Value, 64) +func sNumber(num interface{}, p uint) string { + base, err := strconv.ParseFloat(fmt.Sprintf("%v", num), 64) if err != nil { - v.Value = "0" - return v - } - - // calculate percision - var p = 0 - if f.Options != nil { - p = int(f.Options.Int64(fieldOpt_Number_precision)) - - if p < fieldOpt_Number_precision_min { - p = fieldOpt_Number_precision_min - } else if p > fieldOpt_Number_precision_max { - p = fieldOpt_Number_precision_max - } + return "0" } // Format the value to the desired precision - v.Value = strconv.FormatFloat(base, 'f', p, 64) + str := strconv.FormatFloat(base, 'f', int(p), 64) // In case of fractures, remove trailing 0's - if strings.Contains(v.Value, ".") { - v.Value = strings.TrimRight(v.Value, "0") - v.Value = strings.TrimRight(v.Value, ".") + if strings.Contains(str, ".") { + str = strings.TrimRight(str, "0") + str = strings.TrimRight(str, ".") } - return v + return str } -// -//func (sanitizer) sEmail(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// // @todo extract from "name" format -// return v -//} -// -//func (sanitizer) sFile(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// return v -//} -// -// -//func (sanitizer) sRecord(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// return v -//} -// -//func (sanitizer) sSelect(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// return v -//} -// -//func (sanitizer) sString(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// return v -//} -// -//func (sanitizer) sUrl(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// return v -//} -// -//func (sanitizer) sUser(v *types.RecordValue, f *types.ModuleField, m *types.Module) *types.RecordValue { -// return v -//} +// sanitize casts value to field kind format +func sanitize(f *types.ModuleField, v interface{}) string { + switch strings.ToLower(f.Kind) { + case "bool": + return sBool(v) + case "datetime": + v = sDatetime(v, f.Options.Bool("onlyDate"), f.Options.Bool("onlyTime")) + case "number": + v = sNumber(v, f.Options.Precision()) + } + + return fmt.Sprintf("%v", v) +} diff --git a/compose/service/values/sanitizer_test.go b/compose/service/values/sanitizer_test.go index 5f3fd99ee..0b9574f44 100644 --- a/compose/service/values/sanitizer_test.go +++ b/compose/service/values/sanitizer_test.go @@ -137,10 +137,10 @@ func Test_sanitizer_Run(t *testing.T) { output: "42.555556", }, { - name: "number precision; clamped between [0, 6]", + name: "number precision; round", kind: "Number", - options: map[string]interface{}{"precision": -1}, - input: "42.4", + options: map[string]interface{}{"precision": 0}, + input: "41.6", output: "42", }, { diff --git a/compose/service/values/shared.go b/compose/service/values/shared.go index 211b68d63..f045c2e0b 100644 --- a/compose/service/values/shared.go +++ b/compose/service/values/shared.go @@ -18,10 +18,6 @@ const ( fieldOpt_Datetime_onlyFutureValues = "onlyFutureValues" fieldOpt_Datetime_onlyPastValues = "onlyPastValues" - fieldOpt_Number_precision = "precision" - fieldOpt_Number_precision_min = 0 - fieldOpt_Number_precision_max = 6 - fieldOpt_Url_onlySecure = "onlySecure" ) diff --git a/compose/service/values/validator.go b/compose/service/values/validator.go index 5faf562d5..e6787b307 100644 --- a/compose/service/values/validator.go +++ b/compose/service/values/validator.go @@ -273,11 +273,7 @@ func (vldtr validator) vFile(ctx context.Context, s store.Storer, v *types.Recor } func (vldtr validator) vNumber(v *types.RecordValue, f *types.ModuleField, r *types.Record, m *types.Module) []types.RecordValueError { - var ( - precision = uint(f.Options.Int64Def(fieldOpt_Number_precision, 2)) - ) - - if _, _, err := big.ParseFloat(v.Value, 0, precision, big.ToNearestEven); err != nil { + if _, _, err := big.ParseFloat(v.Value, 0, f.Options.Precision(), big.ToNearestEven); err != nil { return e2s(makeInvalidValueErr(f, v.Value)) } diff --git a/compose/types/module_field.go b/compose/types/module_field.go index f3752db7d..a135dc26c 100644 --- a/compose/types/module_field.go +++ b/compose/types/module_field.go @@ -28,6 +28,8 @@ type ( Multi bool `json:"isMulti"` DefaultValue RecordValueSet `json:"defaultValue"` + Expressions ModuleFieldExpr `json:"expressions"` + Labels map[string]string `json:"labels,omitempty"` CreatedAt time.Time `json:"createdAt,omitempty"` @@ -39,6 +41,16 @@ type ( ModuleID []uint64 Deleted filter.State } + + ModuleFieldExpr struct { + Value string `json:"value,omitempty"` + //Sanitizers []string `json:"sanitizers,omitempty"` + //Validators []struct { + // Test string `json:"test,omitempty"` + // Error string `json:"error,omitempty"` + //} `json:"validators,omitempty"` + //Formatters []string `json:"formatters,omitempty"` + } ) var ( diff --git a/compose/types/module_field_options.go b/compose/types/module_field_options.go index 36b5b3185..8a4349835 100644 --- a/compose/types/module_field_options.go +++ b/compose/types/module_field_options.go @@ -14,8 +14,13 @@ type ( ) const ( + moduleFieldOptionExpression = "expression" moduleFieldOptionIsUnique = "isUnique" moduleFieldOptionIsUniqueMultiValue = "isUniqueMultiValue" + + moduleFieldNumberOptionPrecision = "precision" + moduleFieldNumberOptionPrecisionMin uint = 0 + moduleFieldNumberOptionPrecisionMax uint = 6 ) func (opt *ModuleFieldOptions) Scan(value interface{}) error { @@ -50,6 +55,19 @@ func (opt ModuleFieldOptions) Bool(key string) bool { return false } +// Bool returns option value for key as boolean true or false +// +// Invalid, non-existing are returned as false +func (opt ModuleFieldOptions) String(key string) string { + if _, has := opt[key]; has { + if v, ok := opt[key].(string); ok { + return v + } + } + + return "" +} + func (opt ModuleFieldOptions) Int64(key string) int64 { return opt.Int64Def(key, 0) } @@ -124,3 +142,19 @@ func (opt ModuleFieldOptions) SetIsUniqueMultiValue(value bool) { // SetIsUniqueMultiValue - should value in this field be unique in the multi-value set? opt[moduleFieldOptionIsUniqueMultiValue] = value } + +func (opt ModuleFieldOptions) Precision() (p uint) { + p = uint(opt.Int64(moduleFieldNumberOptionPrecision)) + + if p < moduleFieldNumberOptionPrecisionMin { + p = moduleFieldNumberOptionPrecisionMin + } else if p > moduleFieldNumberOptionPrecisionMax { + p = moduleFieldNumberOptionPrecisionMax + } + + return +} + +func (opt ModuleFieldOptions) SetPrecision(p uint) { + opt[moduleFieldNumberOptionPrecision] = p +} diff --git a/compose/types/record.go b/compose/types/record.go index c27174e92..0a95abd0e 100644 --- a/compose/types/record.go +++ b/compose/types/record.go @@ -118,6 +118,23 @@ func (r Record) DynamicRoles(userID uint64) []uint64 { ) } +func (r Record) Dict(m *Module) map[string]interface{} { + return map[string]interface{}{ + "ID": r.ID, + "moduleID": r.ModuleID, + "values": r.Values.Dict(m.Fields), + "labels": r.Labels, + "namespaceID": r.NamespaceID, + "ownedBy": r.OwnedBy, + "createdAt": r.CreatedAt, + "createdBy": r.CreatedBy, + "updatedAt": r.UpdatedAt, + "updatedBy": r.UpdatedBy, + "deletedAt": r.DeletedAt, + "deletedBy": r.DeletedBy, + } +} + // UnmarshalJSON for custom record deserialization // // Due to https://github.com/golang/go/issues/21092, we should manually reset the given record value set. diff --git a/compose/types/record_value.go b/compose/types/record_value.go index 8c4c3d6e1..90b0011bc 100644 --- a/compose/types/record_value.go +++ b/compose/types/record_value.go @@ -5,7 +5,10 @@ import ( "encoding/json" "fmt" "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/payload" "github.com/pkg/errors" + "strconv" + "strings" "time" ) @@ -82,6 +85,26 @@ func (set RecordValueSet) FilterByRecordID(recordID uint64) (vv RecordValueSet) return } +// Replaces existing values, remove extra +func (set RecordValueSet) Replace(name string, values ...string) (vv RecordValueSet) { + for i := range set { + if set[i].Name != name { + // copy values from other fields + vv = append(vv, set[i]) + } + } + + for p, v := range values { + vv = append(vv, &RecordValue{ + Name: name, + Value: v, + Place: uint(p), + }) + } + + return +} + // Set updates existing value or creates a new one func (set RecordValueSet) Set(v *RecordValue) RecordValueSet { for i := range set { @@ -308,6 +331,53 @@ func (set RecordValueSet) String() (o string) { return o } +// Returns structured representation of values casted to the appropriate types +func (set RecordValueSet) Dict(fields ModuleFieldSet) map[string]interface{} { + var ( + rval = make(map[string]interface{}) + + format = func(f *ModuleField, v string) interface{} { + switch strings.ToLower(f.Kind) { + case "bool": + return payload.ParseBool(v) + case "number": + if f.Options.Precision() > 0 { + num, _ := strconv.ParseFloat(v, 64) + return num + } + + num, _ := strconv.ParseInt(v, 10, 64) + return num + } + + return v + } + ) + + if len(fields) == 0 { + return rval + } + + _ = fields.Walk(func(f *ModuleField) error { + if f.Multi { + var ( + rv = set.FilterByName(f.Name) + vv = make([]interface{}, len(rv)) + ) + for i, val := range rv { + vv[i] = format(f, val.Value) + } + rval[f.Name] = vv + } else if v := set.Get(f.Name, 0); v != nil { + rval[f.Name] = format(f, v.Value) + } + + return nil + }) + + return rval +} + func (set RecordValueSet) Len() int { return len(set) } func (set RecordValueSet) Swap(i, j int) { set[i], set[j] = set[j], set[i] } func (set RecordValueSet) Less(i, j int) bool { return set[i].Place < set[j].Place }