DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Extends DeDup capabilities

Browse Source 
Support modifier for value search and allow ability to select matching criteria for multi value field

- Removed name from rule for now
- Value modifier to search with are ignore-case, case-sensitive, fuzzy-search, sounds-like
- Multi value matching criteria are one-of, equal
- Migrate RecordDeDup config for module, by adding upgrade fix for module.config.recordDeDup to migrate as per to the latest DeDupRule struct.
This commit is contained in:
Vivek Patel
2023-01-25 21:54:36 +05:30
parent 5f4d02ac84
commit e7fa07cfae
11 changed files with 716 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/compose/service/record.go
+41 -19
View File
@@ -571,6 +571,12 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r
// before we start storing any changes
rves = &types.RecordValueErrorSet{}
// duplication errors
ddes = &types.RecordValueErrorSet{}
// merge of record value errors and duplication errors
ee = &types.RecordValueErrorSet{}
action func(props ...*recordActionProps) *recordAction
r *types.Record
@@ -599,11 +605,11 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r
switch p.Operation {
case types.OperationTypeCreate:
action = RecordActionCreate
r, dd, err = svc.create(ctx, r)
r, ddes, err = svc.create(ctx, r)
case types.OperationTypeUpdate:
action = RecordActionUpdate
r, dd, err = svc.update(ctx, r)
r, ddes, err = svc.update(ctx, r)
case types.OperationTypeDelete:
action = RecordActionDelete
@@ -613,8 +619,13 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r
aProp.setChanged(r)
// Attach meta ID to each value error for FE identification
if !dd.HasStrictErrors() && r != nil {
dd.SetMetaID(r.ID)
if !ddes.HasStrictErrors() && r != nil {
ddes.SetMetaID(r.ID)
}
if !ddes.IsValid() && dd == nil {
dd = ddes
} else {
dd.Merge(ddes)
}
if rve := types.IsRecordValueErrorSet(err); rve != nil {
@@ -644,9 +655,14 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r
}
}
if !rves.IsValid() {
// merge record value errors and strict duplication errors
if dd.HasStrictErrors() {
ee.Merge(rves, dd)
}
if !ee.IsValid() {
// Any errors gathered?
return RecordErrValueInput().Wrap(rves)
return RecordErrValueInput().Wrap(ee)
}
return nil
@@ -701,10 +717,19 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec
new.SetModule(m)
{
// handle deDup error/warnings
dd, err = svc.DupDetection(ctx, m, new)
// handle input payload errors
if rve = svc.procCreate(ctx, invokerID, m, new); !rve.IsValid() {
return nil, dd, RecordErrValueInput().Wrap(rve)
}
// record value errors from dup detection
if err != nil {
return
}
if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeCreate(new, nil, m, ns, rve, nil)); err != nil {
return
} else if !rve.IsValid() {
@@ -714,11 +739,6 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec
new.Values = RecordValueDefaults(m, new.Values)
dd, err = svc.DupDetection(ctx, m, new)
if err != nil {
return
}
// Handle payload from automation scripts
if rve = svc.procCreate(ctx, invokerID, m, new); !rve.IsValid() {
return nil, dd, RecordErrValueInput().Wrap(rve)
@@ -996,17 +1016,20 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
upd.SetModule(m)
old.SetModule(m)
dd, err = svc.DupDetection(ctx, m, upd)
if err != nil {
return
}
{
// Handle input payload
// handle deDup error/warnings
dd, err = svc.DupDetection(ctx, m, upd)
// handle input payload errors
if rve = svc.procUpdate(ctx, invokerID, m, upd, old); !rve.IsValid() {
return nil, dd, RecordErrValueInput().Wrap(rve)
}
// record value errors from dup detection
if err != nil {
return
}
// Scripts can (besides simple error value) return complex record value error set
// that is passed back to the UI or any other API consumer
//
@@ -1833,7 +1856,6 @@ func (svc record) DupDetection(ctx context.Context, m *types.Module, rec *types.
return
}
// @todo: improve error string with details
rProps.setValueErrors(out)
// Error out if duplicate record exist
@@ -2036,7 +2058,7 @@ fields:
val.Value = pickRandomID(recRefs[refModID])
case "select":
//val.Value = src.Select(f.Options)
// val.Value = src.Select(f.Options)
continue fields
case "url":
server/compose/types/module.go
+1 -4
View File
@@ -93,12 +93,9 @@ type (
}
ModuleConfigRecordDeDup struct {
// enable or disable duplicate detection
Enabled bool `json:"enabled"`
// strictly restrict record saving
// otherwise show a warning with list of duplicated records
Strict bool `json:"strict"`
Strict bool `json:"-"`
// list of duplicate detection rules applied to module's fields
Rules DeDupRuleSet `json:"rules,omitempty"`
server/compose/types/record_detect_duplicates.go
+109 -51
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/cortezaproject/corteza/server/pkg/locale"
"github.com/cortezaproject/corteza/server/pkg/str"
"github.com/spf13/cast"
"strings"
)
@@ -18,23 +19,44 @@ type (
}
DeDupRule struct {
Name DeDupRuleName `json:"name"`
Strict bool `json:"strict"`
Attributes []string `json:"attributes"`
Name DeDupRuleName `json:"name"`
Strict bool `json:"strict"`
ErrorMessage string `json:"errorMessage"`
ConstraintSet DeDupRuleConstraintSet `json:"constraints"`
}
DeDupRuleConstraint struct {
Attribute string `json:"attribute"`
Modifier DeDupValueModifier `json:"modifier"`
MultiValue DeDupMultiValueConstraint `json:"multiValue"`
}
DeDupRuleConstraintSet []*DeDupRuleConstraint
// DeDupRuleName represent the identifier for duplicate detection rule
DeDupRuleName string
// DeDupValueModifier represent the algorithm used to check value string
DeDupValueModifier string
// DeDupMultiValueConstraint for matching multi values accordingly
DeDupMultiValueConstraint string
// DeDupIssueKind based on strict mode rule or duplication config
DeDupIssueKind string
)
const (
caseSensitive DeDupRuleName = "case-sensitive"
ignoreCase DeDupValueModifier = "ignore-case"
caseSensitive DeDupValueModifier = "case-sensitive"
fuzzyMatch DeDupValueModifier = "fuzzy-match"
soundsLike DeDupValueModifier = "sounds-like"
dupWarning DeDupIssueKind = "duplication_warning"
dupError DeDupIssueKind = "duplication_error"
oneOf DeDupMultiValueConstraint = "one-of"
equal DeDupMultiValueConstraint = "equal"
deDupWarning DeDupIssueKind = "duplication_warning"
deDupError DeDupIssueKind = "duplication_error"
)
func DeDup() *deDup {
@@ -47,7 +69,7 @@ func (d deDup) CheckDuplication(ctx context.Context, rules DeDupRuleSet, rec Rec
out = &RecordValueErrorSet{}
err = rules.Walk(func(rule *DeDupRule) error {
if rule.HasAttributes() {
values := rr.GetValuesByName(distinct(rule.Attributes)...)
values := rr.GetValuesByName(distinct(rule.Attributes())...)
set := rule.validateValue(ctx, d.ls, rec, values)
@@ -72,7 +94,14 @@ func (rule DeDupIssueKind) String() string {
}
func (rule DeDupRule) HasAttributes() bool {
return len(rule.Attributes) > 0
return len(rule.ConstraintSet) > 0 && len(rule.Attributes()) > 0
}
func (rule DeDupRule) Attributes() (out []string) {
for _, c := range rule.ConstraintSet {
out = append(out, c.Attribute)
}
return
}
func (rule DeDupRule) IsStrict() bool {
@@ -80,9 +109,9 @@ func (rule DeDupRule) IsStrict() bool {
}
func (rule DeDupRule) IssueKind() string {
out := dupWarning
out := deDupWarning
if rule.Strict {
out = dupError
out = deDupError
}
return out.String()
@@ -93,43 +122,56 @@ func (rule DeDupRule) IssueMessage() (out string) {
}
func (rule DeDupRule) String() string {
return fmt.Sprintf("%s duplicate detection on `%s` field", rule.Name, strings.Join(rule.Attributes, ", "))
return fmt.Sprintf("%s duplicate detection on `%s` field", rule.Name, strings.Join(rule.Attributes(), ", "))
}
// validateValue will check duplicate detection based on rules name
func (rule DeDupRule) validateValue(ctx context.Context, ls localeService, rec Record, vv RecordValueSet) (out *RecordValueErrorSet) {
switch rule.Name {
case caseSensitive:
return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv)
default:
return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv)
}
return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv)
}
func (rule DeDupRule) checkCaseSensitiveDuplication(ctx context.Context, ls localeService, rec Record, vv RecordValueSet) (out *RecordValueErrorSet) {
out = &RecordValueErrorSet{}
recVal := rec.Values
var (
recVal = rec.Values
)
for _, a := range rule.Attributes {
rv := recVal.Get(a, 0)
if rv == nil {
for _, c := range rule.ConstraintSet {
rvv := recVal.FilterByName(c.Attribute)
if rvv.Len() == 0 {
continue
}
var (
valErr = &RecordValueErrorSet{}
)
_ = vv.Walk(func(v *RecordValue) error {
if v.RecordID != rec.ID {
if toLower(v.Value) == toLower(rv.Value) {
out.Push(RecordValueError{
Kind: rule.IssueKind(),
Message: ls.T(ctx, "compose", rule.IssueMessage()),
Meta: map[string]interface{}{
"field": v.Name,
"value": v.Value,
"dupValueField": rv.Name,
"recordID": cast.ToString(v.RecordID),
"rule": rule.String(),
},
})
_ = rvv.Walk(func(rv *RecordValue) error {
if len(rv.Value) > 0 && matchValue(c.Modifier, rv.Value, v.Value) {
valErr.Push(RecordValueError{
Kind: rule.IssueKind(),
Message: ls.T(ctx, "compose", rule.IssueMessage()),
Meta: map[string]interface{}{
"field": v.Name,
"value": v.Value,
"dupValueField": rv.Name,
"recordID": cast.ToString(v.RecordID),
"rule": rule.String(),
},
})
}
return nil
})
// 1. multiValue is empty, then all value needs to be a match then return error/warning
// 2. multiValue is oneOf, then one or more value needs to be a match then return error/warning
// 3. multiValue is equal, then all value needs to be a match then return error/warning
if (!valErr.IsValid() && (!c.HasMultiValue() || c.IsAllEqual()) && valErr.Len() == rvv.Len()) || (c.IsOneOf() && valErr.Len() > 0) {
if out == nil {
out = &RecordValueErrorSet{}
}
out.Push(valErr.Set...)
}
}
return nil
@@ -139,6 +181,23 @@ func (rule DeDupRule) checkCaseSensitiveDuplication(ctx context.Context, ls loca
return
}
func (c DeDupRuleConstraint) HasMultiValue() bool {
switch c.MultiValue {
case oneOf, equal:
return true
default:
return false
}
}
func (c DeDupRuleConstraint) IsAllEqual() bool {
return c.MultiValue == equal
}
func (c DeDupRuleConstraint) IsOneOf() bool {
return c.MultiValue == oneOf
}
func (v *RecordValueErrorSet) SetMetaID(id uint64) {
if v.IsValid() {
return
@@ -154,21 +213,7 @@ func (v *RecordValueErrorSet) SetMetaID(id uint64) {
}
func (v *RecordValueErrorSet) HasStrictErrors() bool {
return v.HasKind(dupError.String())
}
// CaseSensitiveDuplicationRule prepares the case-sensitive duplicate detection rule
func CaseSensitiveDuplicationRule(strict bool, identifiers ...string) DeDupRule {
return makeDuplicationRule(caseSensitive, strict, identifiers...)
}
// makeDuplicationRule prepares duplication detection rules
func makeDuplicationRule(name DeDupRuleName, strict bool, attributes ...string) DeDupRule {
return DeDupRule{
Name: name,
Strict: strict,
Attributes: attributes,
}
return v.HasKind(deDupError.String())
}
// distinct only list the different (distinct) values
@@ -183,6 +228,19 @@ func distinct(input []string) (out []string) {
return
}
func toLower(s string) string {
return strings.ToLower(s)
// matchValue will check if the input matches with target string as per the modifier
func matchValue(modifier DeDupValueModifier, input string, target string) bool {
switch modifier {
case ignoreCase:
return str.Match(input, target, str.CaseInSensitiveMatch)
case caseSensitive:
return str.Match(input, target, str.CaseSensitiveMatch)
case fuzzyMatch:
return str.Match(input, target, str.LevenshteinDistance)
case soundsLike:
return str.Match(input, target, str.Soundex)
default:
// ignoreCase as default, if not specified
return str.Match(input, target, str.CaseInSensitiveMatch)
}
}
server/compose/types/record_detect_duplicates_test.go
+126
View File
@@ -0,0 +1,126 @@
package types
import (
"context"
"github.com/cortezaproject/corteza/server/pkg/locale"
"github.com/spf13/cast"
"github.com/stretchr/testify/require"
"testing"
)
func TestDeDupRule_checkCaseSensitiveDuplication(t *testing.T) {
var (
req = require.New(t)
ctx = context.Background()
ls = locale.Global()
rule1 = DeDupRule{
Name: "",
Strict: true,
ConstraintSet: []*DeDupRuleConstraint{
{
Attribute: "name",
Modifier: ignoreCase,
},
},
}
tests = []struct {
name string
rule DeDupRule
rec Record
vv RecordValueSet
wantOut *RecordValueErrorSet
}{
{
name: "no duplication",
rule: rule1,
rec: Record{
ID: 1,
Values: RecordValueSet{
&RecordValue{
RecordID: 1,
Name: "name",
Value: "test",
},
},
},
vv: RecordValueSet{
&RecordValue{
RecordID: 2,
Name: "name",
Value: "test",
},
},
wantOut: &RecordValueErrorSet{
Set: []RecordValueError{
{
Kind: deDupError.String(),
Message: rule1.IssueMessage(),
Meta: map[string]interface{}{
"field": "name",
"value": "test",
"dupValueField": "name",
"recordID": cast.ToString(2),
"rule": rule1.String(),
},
},
},
},
},
}
)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := tt.rule.checkCaseSensitiveDuplication(ctx, ls, tt.rec, tt.vv)
req.Equal(tt.wantOut, gotOut, "checkCaseSensitiveDuplication() = %v, want %v", gotOut, tt.wantOut)
})
}
}
func Test_matchValue(t *testing.T) {
tests := []struct {
name string
input string
target string
modifier DeDupValueModifier
want bool
}{
{
name: "ignoreCase match value",
input: "test",
target: "tEst",
modifier: ignoreCase,
want: true,
},
{
name: "caseSensitive match value",
input: "tEst",
target: "tEst",
modifier: caseSensitive,
want: true,
},
{
name: "fuzzyMatch match value",
input: "kitten",
target: "sitting",
modifier: fuzzyMatch,
want: true,
},
{
name: "soundsLike match value",
input: "Robert",
target: "Rupert",
modifier: soundsLike,
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchValue(tt.modifier, tt.input, tt.target); got != tt.want {
t.Errorf("matchValue() = %v, want %v", got, tt.want)
}
})
}
}
server/compose/types/validated.go
+21
View File
@@ -29,6 +29,13 @@ func (v *RecordValueErrorSet) IsValid() bool {
return v == nil || len(v.Set) == 0
}
func (v *RecordValueErrorSet) Len() int {
if v == nil {
return 0
}
return len(v.Set)
}
func (v *RecordValueErrorSet) Error() string {
var no = 0
if v != nil {
@@ -62,6 +69,20 @@ func (v *RecordValueErrorSet) HasKind(kind string) bool {
return false
}
func (v *RecordValueErrorSet) Merge(errs ...*RecordValueErrorSet) {
if v == nil {
return
}
for _, e := range errs {
if e == nil || e.IsValid() {
continue
}
v.Push(e.Set...)
}
}
// IsRecordValueErrorSet tests if given error is RecordValueErrorSet (or it wraps it) and it has errors
// If not is not (or !IsValid), it return nil!
func IsRecordValueErrorSet(err error) *RecordValueErrorSet {