DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Add duplicate detection for compose record values

Browse Source 
For now only case-sensitive detection is supported, there are 2 type of duplicate detection, one is strict mode which will throw an error on record save if there is any duplicate record with matching value and non-strict mode will only show a warning and won't prevent record saving.
This commit is contained in:
Vivek Patel
2022-09-21 19:24:43 +05:30
parent b3cdbd060f
commit 9d46f5c234
18 changed files with 471 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/automation/records_handler.go
+8 -7
View File
@@ -13,12 +13,12 @@ import (
type (
recordService interface {
FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, error)
FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, *types.RecordValueErrorSet, error)
Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error)
Create(ctx context.Context, record *types.Record) (*types.Record, error)
Update(ctx context.Context, record *types.Record) (*types.Record, error)
Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, error)
Create(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error)
Update(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error)
Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, *types.RecordValueErrorSet, error)
Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (out any, err error)
Validate(ctx context.Context, rec *types.Record) error
@@ -273,13 +273,13 @@ func (h recordsHandler) new(ctx context.Context, args *recordsNewArgs) (*records
func (h recordsHandler) create(ctx context.Context, args *recordsCreateArgs) (results *recordsCreateResults, err error) {
results = &recordsCreateResults{}
results.Record, err = h.rec.Create(ctx, args.Record)
results.Record, _, err = h.rec.Create(ctx, args.Record)
return
}
func (h recordsHandler) update(ctx context.Context, args *recordsUpdateArgs) (results *recordsUpdateResults, err error) {
results = &recordsUpdateResults{}
results.Record, err = h.rec.Update(ctx, args.Record)
results.Record, _, err = h.rec.Update(ctx, args.Record)
return
}
@@ -322,7 +322,8 @@ func (h recordsHandler) lookupRecord(ctx context.Context, args recordLookup) (re
return
}
return h.rec.FindByID(ctx, namespace.ID, module.ID, recordID)
record, _, err = h.rec.FindByID(ctx, namespace.ID, module.ID, recordID)
return
}
func (h recordsHandler) loadCombo(ctx context.Context, args interface{}) (namespace *types.Namespace, module *types.Module, err error) {
compose/rest/record.go
+17 -15
View File
@@ -33,7 +33,8 @@ type (
recordPayload struct {
*types.Record
Records types.RecordSet `json:"records,omitempty"`
Records types.RecordSet `json:"records,omitempty"`
RecordValueErrors *types.RecordValueErrorSet `json:"valueErrors"`
CanManageOwnerOnRecord bool `json:"canManageOwnerOnRecord"`
CanUpdateRecord bool `json:"canUpdateRecord"`
@@ -147,14 +148,14 @@ func (ctrl *Record) Read(ctx context.Context, r *request.RecordRead) (interface{
return nil, err
}
record, err := ctrl.record.FindByID(ctx, r.NamespaceID, r.ModuleID, r.RecordID)
record, dd, err := ctrl.record.FindByID(ctx, r.NamespaceID, r.ModuleID, r.RecordID)
// Temp workaround until we do proper by-module filtering for record findByID
if record != nil && record.ModuleID != r.ModuleID {
return nil, store.ErrNotFound
}
return ctrl.makePayload(ctx, m, record, err)
return ctrl.makePayload(ctx, m, record, dd, err)
}
func (ctrl *Record) Create(ctx context.Context, r *request.RecordCreate) (interface{}, error) {
@@ -200,12 +201,12 @@ func (ctrl *Record) Create(ctx context.Context, r *request.RecordCreate) (interf
}
oo = append(oo, oob...)
rr, err := ctrl.record.Bulk(ctx, oo...)
rr, dd, err := ctrl.record.Bulk(ctx, oo...)
if rve := types.IsRecordValueErrorSet(err); rve != nil {
return ctrl.handleValidationError(rve), nil
}
return ctrl.makeBulkPayload(ctx, m, err, rr...)
return ctrl.makeBulkPayload(ctx, m, dd, err, rr...)
}
func (ctrl *Record) Update(ctx context.Context, r *request.RecordUpdate) (interface{}, error) {
@@ -252,13 +253,12 @@ func (ctrl *Record) Update(ctx context.Context, r *request.RecordUpdate) (interf
}
oo = append(oo, oob...)
rr, err := ctrl.record.Bulk(ctx, oo...)
rr, dd, err := ctrl.record.Bulk(ctx, oo...)
if rve := types.IsRecordValueErrorSet(err); rve != nil {
return ctrl.handleValidationError(rve), nil
}
return ctrl.makeBulkPayload(ctx, m, err, rr...)
return ctrl.makeBulkPayload(ctx, m, dd, err, rr...)
}
func (ctrl *Record) Delete(ctx context.Context, r *request.RecordDelete) (interface{}, error) {
@@ -550,7 +550,7 @@ func (ctrl *Record) TriggerScript(ctx context.Context, r *request.RecordTriggerS
module, record, err := ctrl.record.TriggerScript(ctx, r.NamespaceID, r.ModuleID, r.RecordID, r.Values, r.Script)
// Script can return modified record and we'll pass it on to the caller
return ctrl.makePayload(ctx, module, record, err)
return ctrl.makePayload(ctx, module, record, nil, err)
}
func (ctrl *Record) TriggerScriptOnList(ctx context.Context, r *request.RecordTriggerScriptOnList) (rsp interface{}, err error) {
@@ -604,14 +604,15 @@ func (ctrl *Record) Revisions(ctx context.Context, r *request.RecordRevisions) (
}, err
}
func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err error, rr ...*types.Record) (*recordPayload, error) {
func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, dd *types.RecordValueErrorSet, err error, rr ...*types.Record) (*recordPayload, error) {
if err != nil || rr == nil {
return nil, err
}
return &recordPayload{
Record: rr[0],
Records: rr[1:],
Record: rr[0],
Records: rr[1:],
RecordValueErrors: dd,
CanManageOwnerOnRecord: ctrl.ac.CanManageOwnerOnRecord(ctx, rr[0]),
CanUpdateRecord: ctrl.ac.CanUpdateRecord(ctx, rr[0]),
@@ -621,13 +622,14 @@ func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err err
}, nil
}
func (ctrl Record) makePayload(ctx context.Context, m *types.Module, r *types.Record, err error) (*recordPayload, error) {
func (ctrl Record) makePayload(ctx context.Context, m *types.Module, r *types.Record, dd *types.RecordValueErrorSet, err error) (*recordPayload, error) {
if err != nil || r == nil {
return nil, err
}
return &recordPayload{
Record: r,
Record: r,
RecordValueErrors: dd,
CanGrant: ctrl.ac.CanGrant(ctx),
@@ -647,7 +649,7 @@ func (ctrl Record) makeFilterPayload(ctx context.Context, m *types.Module, rr ty
modp := &recordSetPayload{Filter: f, Set: make([]*recordPayload, len(rr))}
for i := range rr {
modp.Set[i], _ = ctrl.makePayload(ctx, m, rr[i], nil)
modp.Set[i], _ = ctrl.makePayload(ctx, m, rr[i], nil, nil)
}
return modp, nil
compose/service/record.go
+99 -37
View File
@@ -51,9 +51,10 @@ type (
revisions *recordRevisions
formatter recordValuesFormatter
sanitizer recordValuesSanitizer
validator recordValuesValidator
formatter recordValuesFormatter
sanitizer recordValuesSanitizer
validator recordValuesValidator
dupDetector recordValuesDupDetector
}
recordValuesFormatter interface {
@@ -72,6 +73,10 @@ type (
UserRefChecker(fn values.ReferenceChecker)
}
recordValuesDupDetector interface {
CheckDuplication(context.Context, types.DeDupRuleSet, types.Record, types.RecordSet) (*types.RecordValueErrorSet, error)
}
recordValueAccessController interface {
CanReadRecordValueOnModuleField(context.Context, *types.ModuleField) bool
CanUpdateRecordValueOnModuleField(context.Context, *types.ModuleField) bool
@@ -105,7 +110,7 @@ type (
}
RecordService interface {
FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, error)
FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, *types.RecordValueErrorSet, error)
Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (any, error)
Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error)
@@ -114,9 +119,9 @@ type (
RecordExport(context.Context, types.RecordFilter) error
RecordImport(context.Context, error) error
Create(ctx context.Context, record *types.Record) (*types.Record, error)
Update(ctx context.Context, record *types.Record) (*types.Record, error)
Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, error)
Create(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error)
Update(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error)
Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, *types.RecordValueErrorSet, error)
Validate(ctx context.Context, rec *types.Record) error
@@ -185,8 +190,9 @@ func Record() *record {
revisions: &recordRevisions{revisions.Service(dal.Service())},
formatter: values.Formatter(),
sanitizer: values.Sanitizer(),
formatter: values.Formatter(),
sanitizer: values.Sanitizer(),
dupDetector: types.DeDup(),
}
svc.validator = defaultValidator(svc)
@@ -212,7 +218,7 @@ func defaultValidator(svc RecordService) recordValuesValidator {
return false, nil
}
r, err := svc.FindByID(ctx, f.NamespaceID, f.ModuleID, v.Ref)
r, _, err := svc.FindByID(ctx, f.NamespaceID, f.ModuleID, v.Ref)
return r != nil, err
})
@@ -234,7 +240,7 @@ func defaultValidator(svc RecordService) recordValuesValidator {
}
// lookup fn() orchestrates record lookup, namespace preload and check
func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, lookup func(*types.Module, *recordActionProps) (*types.Record, error)) (r *types.Record, err error) {
func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, lookup func(*types.Module, *recordActionProps) (*types.Record, error)) (r *types.Record, dd *types.RecordValueErrorSet, err error) {
var (
ns *types.Namespace
m *types.Module
@@ -266,13 +272,15 @@ func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, look
r.SetModule(m)
r.Values = svc.sanitizer.RunXSS(m, r.Values)
dd, err = svc.DupDetection(ctx, m, r)
return nil
}()
return r, svc.recordAction(ctx, aProps, RecordActionLookup, err)
return r, dd, svc.recordAction(ctx, aProps, RecordActionLookup, err)
}
func (svc record) FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (r *types.Record, err error) {
func (svc record) FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (r *types.Record, dd *types.RecordValueErrorSet, err error) {
return svc.lookup(ctx, namespaceID, moduleID, func(m *types.Module, props *recordActionProps) (*types.Record, error) {
props.record.ID = recordID
@@ -509,7 +517,7 @@ func (svc record) RecordExport(ctx context.Context, f types.RecordFilter) (err e
// Bulk handles provided set of bulk record operations.
// It's able to create, update or delete records in a single transaction.
func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (rr types.RecordSet, err error) {
func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (rr types.RecordSet, dd *types.RecordValueErrorSet, err error) {
var pr *types.Record
err = func() error {
@@ -563,11 +571,11 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r
switch p.Operation {
case types.OperationTypeCreate:
action = RecordActionCreate
r, err = svc.create(ctx, r)
r, dd, err = svc.create(ctx, r)
case types.OperationTypeUpdate:
action = RecordActionUpdate
r, err = svc.update(ctx, r)
r, dd, err = svc.update(ctx, r)
case types.OperationTypeDelete:
action = RecordActionDelete
@@ -614,20 +622,20 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r
if len(oo) == 1 {
// was not really a bulk operation, and we already recorded the action
// inside transaction loop
return rr, err
return rr, dd, err
} else {
// when doing bulk op (updating and/or creating more than one record at once),
// we already log action for each operation
//
// to log the fact that the bulk op was done, we do one additional recording
// without any props
return rr, svc.recordAction(ctx, &recordActionProps{}, RecordActionBulk, err)
return rr, dd, svc.recordAction(ctx, &recordActionProps{}, RecordActionBulk, err)
}
}
// Raw create function that is responsible for value validation, event dispatching
// and creation.
func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Record, err error) {
func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) {
var (
aProps = &recordActionProps{record: new}
invokerID = auth.GetIdentityFromContext(ctx).Identity()
@@ -645,7 +653,7 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec
aProps.setModule(m)
if !svc.ac.CanCreateRecordOnModule(ctx, m) {
return nil, RecordErrNotAllowedToCreate()
return nil, dd, RecordErrNotAllowedToCreate()
}
if err = RecordValueSanitization(m, new.Values); err != nil {
@@ -661,21 +669,26 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec
{
if rve = svc.procCreate(ctx, invokerID, m, new); !rve.IsValid() {
return nil, RecordErrValueInput().Wrap(rve)
return nil, dd, RecordErrValueInput().Wrap(rve)
}
if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeCreate(new, nil, m, ns, rve, nil)); err != nil {
return
} else if !rve.IsValid() {
return nil, RecordErrValueInput().Wrap(rve)
return nil, dd, RecordErrValueInput().Wrap(rve)
}
}
new.Values = RecordValueDefaults(m, new.Values)
dd, err = svc.DupDetection(ctx, m, new)
if err != nil {
return
}
// Handle payload from automation scripts
if rve = svc.procCreate(ctx, invokerID, m, new); !rve.IsValid() {
return nil, RecordErrValueInput().Wrap(rve)
return nil, dd, RecordErrValueInput().Wrap(rve)
}
aProps.setChanged(new)
@@ -828,6 +841,7 @@ func CalcRecordOwner(current, new, invoker uint64) uint64 {
return new
}
// @todo: ?? this might be a good place for detection too
func RecordValueUpdateOpCheck(ctx context.Context, ac recordValueAccessController, m *types.Module, vv types.RecordValueSet) *types.RecordValueErrorSet {
rve := &types.RecordValueErrorSet{}
if ac == nil {
@@ -905,7 +919,7 @@ func RecordValueDefaults(m *types.Module, vv types.RecordValueSet) (out types.Re
// Raw update function that is responsible for value validation, event dispatching
// and update.
func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Record, err error) {
func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) {
var (
aProps = &recordActionProps{record: upd}
invokerID = auth.GetIdentityFromContext(ctx).Identity()
@@ -916,7 +930,7 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
)
if upd.ID == 0 {
return nil, RecordErrInvalidID()
return nil, dd, RecordErrInvalidID()
}
ns, m, old, err = loadRecordCombo(ctx, svc.store, svc.dal, upd.NamespaceID, upd.ModuleID, upd.ID)
@@ -929,12 +943,12 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
aProps.setRecord(old)
if !svc.ac.CanUpdateRecord(ctx, old) {
return nil, RecordErrNotAllowedToUpdate()
return nil, dd, RecordErrNotAllowedToUpdate()
}
// Test if stale (update has an older version of data)
if isStale(upd.UpdatedAt, old.UpdatedAt, old.CreatedAt) {
return nil, RecordErrStaleData()
return nil, dd, RecordErrStaleData()
}
if err = RecordValueSanitization(m, upd.Values); err != nil {
@@ -949,10 +963,15 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
upd.SetModule(m)
old.SetModule(m)
dd, err = svc.DupDetection(ctx, m, upd)
if err != nil {
return
}
{
// Handle input payload
if rve = svc.procUpdate(ctx, invokerID, m, upd, old); !rve.IsValid() {
return nil, RecordErrValueInput().Wrap(rve)
return nil, dd, RecordErrValueInput().Wrap(rve)
}
// Scripts can (besides simple error value) return complex record value error set
@@ -963,13 +982,13 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeUpdate(upd, old, m, ns, rve, nil)); err != nil {
return
} else if !rve.IsValid() {
return nil, RecordErrValueInput().Wrap(rve)
return nil, dd, RecordErrValueInput().Wrap(rve)
}
}
// Handle payload from automation scripts
if rve = svc.procUpdate(ctx, invokerID, m, upd, old); !rve.IsValid() {
return nil, RecordErrValueInput().Wrap(rve)
return nil, dd, RecordErrValueInput().Wrap(rve)
}
err = store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) error {
@@ -990,7 +1009,7 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
})
if err != nil {
return nil, err
return nil, dd, err
}
// ensure module ref is set before running through records workflows and scripts
@@ -1012,18 +1031,18 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
return
}
func (svc record) Create(ctx context.Context, new *types.Record) (rec *types.Record, err error) {
func (svc record) Create(ctx context.Context, new *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) {
var (
aProps = &recordActionProps{record: new}
)
err = func() error {
rec, err = svc.create(ctx, new)
rec, dd, err = svc.create(ctx, new)
aProps.setRecord(rec)
return err
}()
return rec, svc.recordAction(ctx, aProps, RecordActionCreate, err)
return rec, dd, svc.recordAction(ctx, aProps, RecordActionCreate, err)
}
// Runs value sanitization, sets values that should be used
@@ -1079,18 +1098,18 @@ func (svc record) procCreate(ctx context.Context, invokerID uint64, m *types.Mod
return rve
}
func (svc record) Update(ctx context.Context, upd *types.Record) (rec *types.Record, err error) {
func (svc record) Update(ctx context.Context, upd *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) {
var (
aProps = &recordActionProps{record: upd}
)
err = func() error {
rec, err = svc.update(ctx, upd)
rec, dd, err = svc.update(ctx, upd)
aProps.setRecord(rec)
return err
}()
return rec, svc.recordAction(ctx, aProps, RecordActionUpdate, err)
return rec, dd, svc.recordAction(ctx, aProps, RecordActionUpdate, err)
}
// Runs value sanitization, copies values that should updated
@@ -1628,6 +1647,49 @@ func (svc record) Iterator(ctx context.Context, f types.RecordFilter, fn eventbu
return svc.recordAction(ctx, aProps, RecordActionIteratorInvoked, err)
}
// DupDetection check for any duplicate records and returns error for strict duplication
func (svc record) DupDetection(ctx context.Context, m *types.Module, rec *types.Record) (out *types.RecordValueErrorSet, err error) {
if m == nil || rec == nil {
return
}
// @todo: improve per records duplicate detection,
// since it is bit too extreme besides we do have bulk operations
var (
records types.RecordSet
rProps = &recordActionProps{}
config = m.Config.RecordDeDup
)
if config.Enabled {
records, _, err = svc.Find(ctx, types.RecordFilter{
ModuleID: m.ID,
NamespaceID: m.NamespaceID,
})
if err != nil {
return
}
out, err = svc.dupDetector.CheckDuplication(ctx, config.Rules, *rec, records)
if err != nil {
return
}
// @todo: improve error string with details
rProps.setValueErrors(out)
// Error out if duplicate record exist
if (config.Strict && !out.IsValid()) || out.HasStrictErrors() {
return out, types.IsRecordValueErrorSet(out)
} else {
return out, nil
}
}
return
}
func ComposeRecordFilterChecker(ctx context.Context, ac recordAccessController, m *types.Module) func(*types.Record) (bool, error) {
return func(rec *types.Record) (bool, error) {
// Setting module right before we do access control
compose/service/record_test.go
+22 -22
View File
@@ -323,13 +323,13 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) {
// security context w/ writer role
ctx = auth.SetIdentityToContext(ctx, auth.Authenticated(u.ID, writerRole.ID, authRoleID))
recChecked, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valChecked})
recChecked, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valChecked})
verifyRecErrSet(t, err)
req.NotNil(recChecked.Values.Get("bool", 0), "should be checked")
req.Equal("1", recChecked.Values.Get("bool", 0).Value)
recUnchecked, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valUnchecked})
recUnchecked, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valUnchecked})
req.NoError(err)
req.Nil(recUnchecked.Values.Get("bool", 0))
@@ -343,7 +343,7 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) {
&types.RecordValue{Name: "string", Value: "abc"},
}
recChecked, err = svc.Update(ctx, recChecked)
recChecked, _, err = svc.Update(ctx, recChecked)
req.NoError(err)
req.NotNil(recChecked.Values.Get("bool", 0), "should still be checked")
@@ -353,7 +353,7 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) {
&types.RecordValue{Name: "string", Value: "abc"},
}
recUnchecked, err = svc.Update(ctx, recUnchecked)
recUnchecked, _, err = svc.Update(ctx, recUnchecked)
req.NoError(err)
req.Nil(recUnchecked.Values.Get("bool", 0), "should not be checked anymore")
@@ -367,7 +367,7 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) {
&types.RecordValue{Name: "bool", Value: "1"},
}
recChecked, err = svc.Update(ctx, recChecked)
recChecked, _, err = svc.Update(ctx, recChecked)
req.NoError(err)
req.NotNil(recChecked.Values.Get("bool", 0), "should checked again")
@@ -462,13 +462,13 @@ func TestRecord_defValueFieldPermissionIssue(t *testing.T) {
ctx = auth.SetIdentityToContext(ctx, auth.Authenticated(user.ID, authRoleID))
recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}})
recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}})
verifyRecErrSet(t, err)
req.Equal("<def-w><def-r>", valueExtractor(recPartial, "writable", "readable"))
t.Log("creating record with w/o editor role (must be able to crate & update record and modify both fields)")
recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{
recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{
&types.RecordValue{Name: "writable", Value: "w"},
&types.RecordValue{Name: "readable", Value: "r"},
}})
@@ -480,7 +480,7 @@ func TestRecord_defValueFieldPermissionIssue(t *testing.T) {
recPartial.Values = types.RecordValueSet{&types.RecordValue{Name: "writable", Value: "w2"}}
recPartial, err = svc.Update(ctx, recPartial)
recPartial, _, err = svc.Update(ctx, recPartial)
verifyRecErrSet(t, err)
req.Equal("<w2><NULL>", valueExtractor(recPartial, "writable", "readable"))
}
@@ -490,13 +490,13 @@ func TestRecord_defValueFieldPermissionIssue(t *testing.T) {
ctx = auth.SetIdentityToContext(ctx, auth.Authenticated(user.ID, authRoleID, editorRole.ID))
recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}})
recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}})
verifyRecErrSet(t, err)
req.Equal("<def-w><def-r>", valueExtractor(recPartial, "writable", "readable"))
t.Log("creating record with editor role (must be able to crate & update record and modify both fields)")
recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{
recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{
// this is the def. value set
&types.RecordValue{Name: "writable", Value: "def-w"},
&types.RecordValue{Name: "readable", Value: "r"},
@@ -582,14 +582,14 @@ func TestRecord_refAccessControl(t *testing.T) {
{
t.Log("creating record on 1st module; should failed because we do not have permissions to create records")
_, err = svc.Create(ctx, mod1rec1)
_, _, err = svc.Create(ctx, mod1rec1)
req.EqualError(err, "not allowed to create records")
t.Logf("granting permissions to create records on this module")
req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, mod1.RbacResource(), "record.create")))
t.Log("retry creating record on 1st module; should fail because we do not have permissions to update field")
_, err = svc.Create(ctx, mod1rec1)
_, _, err = svc.Create(ctx, mod1rec1)
req.Error(err)
req.True(types.IsRecordValueErrorSet(err).HasKind("updateDenied"))
@@ -597,12 +597,12 @@ func TestRecord_refAccessControl(t *testing.T) {
req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, mod1strField.RbacResource(), "record.value.update")))
t.Log("retry creating record on 1st module; should succeed")
mod1rec1, err = svc.Create(ctx, mod1rec1)
mod1rec1, _, err = svc.Create(ctx, mod1rec1)
req.NoError(err)
}
{
t.Log("can record be read")
_, err = svc.FindByID(ctx, mod1rec1.NamespaceID, mod1rec1.ModuleID, mod1rec1.ID)
_, _, err = svc.FindByID(ctx, mod1rec1.NamespaceID, mod1rec1.ModuleID, mod1rec1.ID)
req.EqualError(err, "not allowed to read this record")
}
{
@@ -610,7 +610,7 @@ func TestRecord_refAccessControl(t *testing.T) {
mod2rec1.Values = mod2rec1.Values.Set(&types.RecordValue{Name: "ref", Value: fmt.Sprintf("%d", mod1rec1.ID)})
t.Log("create record on 2nd module with ref to record on the 1st module; must fail, no create perm")
_, err = svc.Create(ctx, mod2rec1)
_, _, err = svc.Create(ctx, mod2rec1)
req.EqualError(err, "not allowed to create records")
t.Log("grant record.create on namespace level")
@@ -620,36 +620,36 @@ func TestRecord_refAccessControl(t *testing.T) {
req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, types.ModuleFieldRbacResource(ns.ID, 0, 0), "record.value.update")))
t.Log("create record on 2nd module with ref to record on the 1st module; most fail, not allowed to read (referenced) mod1rec1")
_, err = svc.Create(ctx, mod2rec1)
_, _, err = svc.Create(ctx, mod2rec1)
req.EqualError(err, "invalid record value input")
t.Log("grant read on record")
req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, mod1rec1.RbacResource(), "read")))
t.Log("create record on 2nd module with ref to record on the 1st module")
mod2rec1, err = svc.Create(ctx, mod2rec1)
mod2rec1, _, err = svc.Create(ctx, mod2rec1)
verifyRecErrSet(t, err)
}
{
t.Log("update record on 2nd module with unchanged values; must fail, no update permissions")
_, err = svc.Update(ctx, mod2rec1)
_, _, err = svc.Update(ctx, mod2rec1)
req.EqualError(err, "not allowed to update this record")
t.Log("grant update on namespace level")
req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, types.RecordRbacResource(ns.ID, 0, 0), "update")))
t.Log("update record on 2nd module with unchanged values")
mod2rec1, err = svc.Update(ctx, mod2rec1)
mod2rec1, _, err = svc.Update(ctx, mod2rec1)
verifyRecErrSet(t, err)
t.Log("update record on 2nd module with unchanged values; unset record value")
mod2rec1.Values = nil
mod2rec1, err = svc.Update(ctx, mod2rec1)
mod2rec1, _, err = svc.Update(ctx, mod2rec1)
verifyRecErrSet(t, err)
t.Log("link 2nd record to 1st one again")
mod2rec1.Values = mod2rec1.Values.Set(&types.RecordValue{Name: "ref", Value: fmt.Sprintf("%d", mod1rec1.ID)})
mod2rec1, err = svc.Update(ctx, mod2rec1)
mod2rec1, _, err = svc.Update(ctx, mod2rec1)
verifyRecErrSet(t, err)
}
{
@@ -658,7 +658,7 @@ func TestRecord_refAccessControl(t *testing.T) {
t.Log("link 2nd record to 1st one again but w/o permissions; must work, value did not change")
mod2rec1.Values = mod2rec1.Values.Set(&types.RecordValue{Name: "ref", Value: fmt.Sprintf("%d", mod1rec1.ID)})
mod2rec1, err = svc.Update(ctx, mod2rec1)
mod2rec1, _, err = svc.Update(ctx, mod2rec1)
verifyRecErrSet(t, err)
}
}
compose/types/module.go
+16
View File
@@ -6,6 +6,7 @@ import (
"time"
discovery "github.com/cortezaproject/corteza-server/discovery/types"
"github.com/cortezaproject/corteza-server/pkg/sql"
"github.com/jmoiron/sqlx/types"
@@ -58,6 +59,9 @@ type (
Discovery discovery.ModuleMeta `json:"discovery"`
RecordRevisions ModuleConfigRecordRevisions `json:"recordRevisions"`
// RecordDeDup value duplicate detection settings
RecordDeDup ModuleConfigRecordDeDup `json:"recordDeDup"`
}
ModuleConfigDAL struct {
@@ -88,6 +92,18 @@ type (
UsageDisclosure string `json:"usageDisclosure"`
}
ModuleConfigRecordDeDup struct {
// enable or disable duplicate detection
Enabled bool `json:"enabled"`
// strictly restrict record saving
// otherwise show a warning with list of duplicated records
Strict bool `json:"strict"`
// list of duplicate detection rules applied to module's fields
Rules DeDupRuleSet `json:"rules,omitempty"`
}
ModuleFilter struct {
ModuleID []uint64 `json:"moduleID"`
NamespaceID uint64 `json:"namespaceID,string"`
compose/types/record.go
+26
View File
@@ -380,3 +380,29 @@ func (set RecordBulkSet) ToBulkOperations(dftModule uint64, dftNamespace uint64)
return
}
// GetValuesByName filters values for records by names
func (set RecordSet) GetValuesByName(names ...string) (out RecordValueSet) {
nameMap := make(map[string]bool)
for _, n := range names {
if len(n) > 0 {
nameMap[n] = true
}
}
err := set.Walk(func(rec *Record) error {
_ = rec.Values.Walk(func(val *RecordValue) error {
if val != nil && nameMap[val.Name] {
val.RecordID = rec.ID
out = append(out, val)
}
return nil
})
return nil
})
if err != nil {
return
}
return
}
compose/types/record_detect_duplicates.go
+167
View File
@@ -0,0 +1,167 @@
package types
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/locale"
"github.com/spf13/cast"
"strings"
)
type (
deDup struct {
ls localeService
}
localeService interface {
T(ctx context.Context, ns, key string, rr ...string) string
}
DeDupRule struct {
Name DeDupRuleName `json:"name"`
Strict bool `json:"strict"`
Attributes []string `json:"attributes"`
}
// DeDupRuleName represent the identifier for duplicate detection rule
DeDupRuleName string
// DeDupIssueKind based on strict mode rule or duplication config
DeDupIssueKind string
)
const (
caseSensitive DeDupRuleName = "case-sensitive"
dupWarning DeDupIssueKind = "duplication_warning"
dupError DeDupIssueKind = "duplication_error"
)
func DeDup() *deDup {
return &deDup{
ls: locale.Global(),
}
}
func (d deDup) CheckDuplication(ctx context.Context, rules DeDupRuleSet, rec Record, rr RecordSet) (out *RecordValueErrorSet, err error) {
out = &RecordValueErrorSet{}
err = rules.Walk(func(rule *DeDupRule) error {
if rule.HasAttributes() {
values := rr.GetValuesByName(distinct(rule.Attributes)...)
set := rule.validateValue(ctx, d.ls, rec, values)
if !set.IsValid() {
out.Push(set.Set...)
}
}
return nil
})
if err != nil {
return
}
return
}
func (rule DeDupIssueKind) String() string {
return string(rule)
}
func (rule DeDupRule) HasAttributes() bool {
return len(rule.Attributes) > 0
}
func (rule DeDupRule) IsStrict() bool {
return rule.Strict
}
func (rule DeDupRule) IssueKind() string {
out := dupWarning
if rule.Strict {
out = dupError
}
return out.String()
}
func (rule DeDupRule) IssueMessage() (out string) {
return "record-field.errors.duplicateValue"
}
func (rule DeDupRule) String() string {
return fmt.Sprintf("%s duplicate detection on `%s` field", rule.Name, strings.Join(rule.Attributes, ", "))
}
// validateValue will check duplicate detection based on rules name
func (rule DeDupRule) validateValue(ctx context.Context, ls localeService, rec Record, vv RecordValueSet) (out *RecordValueErrorSet) {
switch rule.Name {
case caseSensitive:
return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv)
default:
return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv)
}
}
func (rule DeDupRule) checkCaseSensitiveDuplication(ctx context.Context, ls localeService, rec Record, vv RecordValueSet) (out *RecordValueErrorSet) {
out = &RecordValueErrorSet{}
recVal := rec.Values
_ = recVal.Walk(func(newV *RecordValue) error {
_ = vv.Walk(func(v *RecordValue) error {
if v.RecordID != rec.ID {
if toLower(v.Value) == toLower(newV.Value) {
out.Push(RecordValueError{
Kind: rule.IssueKind(),
Message: ls.T(ctx, "compose", rule.IssueMessage()),
Meta: map[string]interface{}{
"dupValueField": newV.Name,
"recordID": cast.ToString(v.RecordID),
"id": cast.ToString(v.RecordID),
"field": v.Name,
"value": v.Value,
"rule": rule.String(),
},
})
}
}
return nil
})
return nil
})
return
}
func (v *RecordValueErrorSet) HasStrictErrors() bool {
return v.HasKind(dupError.String())
}
// CaseSensitiveDuplicationRule prepares the case-sensitive duplicate detection rule
func CaseSensitiveDuplicationRule(strict bool, identifiers ...string) DeDupRule {
return makeDuplicationRule(caseSensitive, strict, identifiers...)
}
// makeDuplicationRule prepares duplication detection rules
func makeDuplicationRule(name DeDupRuleName, strict bool, attributes ...string) DeDupRule {
return DeDupRule{
Name: name,
Strict: strict,
Attributes: attributes,
}
}
// distinct only list the different (distinct) values
func distinct(input []string) (out []string) {
keys := make(map[string]bool)
for _, val := range input {
if _, ok := keys[val]; !ok {
keys[val] = true
out = append(out, val)
}
}
return
}
func toLower(s string) string {
return strings.ToLower(s)
}
compose/types/type_set.gen.go
Generated
+35
View File
@@ -20,6 +20,11 @@ type (
// This type is auto-generated.
ChartSet []*Chart
// DeDupRuleSet slice of DeDupRule
//
// This type is auto-generated.
DeDupRuleSet []*DeDupRule
// ModuleSet slice of Module
//
// This type is auto-generated.
@@ -168,6 +173,36 @@ func (set ChartSet) IDs() (IDs []uint64) {
return
}
// Walk iterates through every slice item and calls w(DeDupRule) err
//
// This function is auto-generated.
func (set DeDupRuleSet) Walk(w func(*DeDupRule) error) (err error) {
for i := range set {
if err = w(set[i]); err != nil {
return
}
}
return
}
// Filter iterates through every slice item, calls f(DeDupRule) (bool, err) and return filtered slice
//
// This function is auto-generated.
func (set DeDupRuleSet) Filter(f func(*DeDupRule) (bool, error)) (out DeDupRuleSet, err error) {
var ok bool
out = DeDupRuleSet{}
for i := range set {
if ok, err = f(set[i]); err != nil {
return
} else if ok {
out = append(out, set[i])
}
}
return
}
// Walk iterates through every slice item and calls w(Module) err
//
// This function is auto-generated.
compose/types/type_set.gen_test.go
Generated
+56
View File
@@ -194,6 +194,62 @@ func TestChartSetIDs(t *testing.T) {
}
}
func TestDeDupRuleSetWalk(t *testing.T) {
var (
value = make(DeDupRuleSet, 3)
req = require.New(t)
)
// check walk with no errors
{
err := value.Walk(func(*DeDupRule) error {
return nil
})
req.NoError(err)
}
// check walk with error
req.Error(value.Walk(func(*DeDupRule) error { return fmt.Errorf("walk error") }))
}
func TestDeDupRuleSetFilter(t *testing.T) {
var (
value = make(DeDupRuleSet, 3)
req = require.New(t)
)
// filter nothing
{
set, err := value.Filter(func(*DeDupRule) (bool, error) {
return true, nil
})
req.NoError(err)
req.Equal(len(set), len(value))
}
// filter one item
{
found := false
set, err := value.Filter(func(*DeDupRule) (bool, error) {
if !found {
found = true
return found, nil
}
return false, nil
})
req.NoError(err)
req.Len(set, 1)
}
// filter error
{
_, err := value.Filter(func(*DeDupRule) (bool, error) {
return false, fmt.Errorf("filter error")
})
req.Error(err)
}
}
func TestModuleSetWalk(t *testing.T) {
var (
value = make(ModuleSet, 3)
compose/types/types.yaml
+2 -1
View File
@@ -15,4 +15,5 @@ types:
noIdField: true
PrivacyModule:
noIdField: true
DeDupRule:
noIdField: true
federation/service/processer_data_test.go
+6 -6
View File
@@ -198,8 +198,8 @@ func TestProcesserData_persist(t *testing.T) {
}
// create success
func (s testRecordServicePersistSuccess) Create(_ context.Context, record *ct.Record) (*ct.Record, error) {
return nil, nil
func (s testRecordServicePersistSuccess) Create(_ context.Context, record *ct.Record) (*ct.Record, *ct.RecordValueErrorSet, error) {
return nil, nil, nil
}
func (s testRecordServicePersistSuccess) Find(_ context.Context, filter ct.RecordFilter) (ct.RecordSet, ct.RecordFilter, error) {
@@ -207,8 +207,8 @@ func (s testRecordServicePersistSuccess) Find(_ context.Context, filter ct.Recor
}
// update success
func (s testRecordServiceUpdateSuccess) Update(_ context.Context, record *ct.Record) (*ct.Record, error) {
return nil, nil
func (s testRecordServiceUpdateSuccess) Update(_ context.Context, record *ct.Record) (*ct.Record, *ct.RecordValueErrorSet, error) {
return nil, nil, nil
}
func (s testRecordServiceUpdateSuccess) Find(_ context.Context, filter ct.RecordFilter) (ct.RecordSet, ct.RecordFilter, error) {
@@ -225,6 +225,6 @@ func (s testRecordServiceDeleteSuccess) Find(_ context.Context, filter ct.Record
}
// create error
func (s testRecordServicePersistError) Create(_ context.Context, record *ct.Record) (*ct.Record, error) {
return nil, errors.New("mocked error")
func (s testRecordServicePersistError) Create(_ context.Context, record *ct.Record) (*ct.Record, *ct.RecordValueErrorSet, error) {
return nil, nil, errors.New("mocked error")
}
federation/service/sync.go
+6 -4
View File
@@ -79,13 +79,15 @@ func (s *Sync) FetchUrl(ctx context.Context, url string) (io.Reader, error) {
}
// CreateRecord wraps the compose Record service Create
func (s *Sync) CreateRecord(ctx context.Context, rec *ct.Record) (*ct.Record, error) {
return s.composeRecordService.Create(ctx, rec)
func (s *Sync) CreateRecord(ctx context.Context, rec *ct.Record) (out *ct.Record, err error) {
out, _, err = s.composeRecordService.Create(ctx, rec)
return
}
// UpdateRecord wraps the compose Record service Update
func (s *Sync) UpdateRecord(ctx context.Context, rec *ct.Record) (*ct.Record, error) {
return s.composeRecordService.Update(ctx, rec)
func (s *Sync) UpdateRecord(ctx context.Context, rec *ct.Record) (out *ct.Record, err error) {
out, _, err = s.composeRecordService.Update(ctx, rec)
return
}
// DeleteRecord wraps the compose Record service Update
pkg/codegen/codegen.go
Generated
+5 -3
View File
@@ -43,9 +43,10 @@ func Proc() {
// workaround because
// filepath.Join merges "*","*" into "**" instead of "*/*"
typeSrcPath = filepath.Join("*"+string(filepath.Separator)+"*", "types.yaml")
typeSrc []string
typeDefs []*typesDef
pkgTypeSrcPath = filepath.Join("*"+string(filepath.Separator)+"*"+string(filepath.Separator)+"*", "types.yaml")
typeSrcPath = filepath.Join("*"+string(filepath.Separator)+"*", "types.yaml")
typeSrc []string
typeDefs []*typesDef
// workaround because
// filepath.Join merges "*","*" into "**" instead of "*/*"
@@ -144,6 +145,7 @@ func Proc() {
output("loaded %d event definitions from %s\n", len(eventSrc), eventSrcPath)
typeSrc = glob(typeSrcPath)
typeSrc = append(typeSrc, glob(pkgTypeSrcPath)...)
output("loaded %d type definitions from %s\n", len(typeSrc), typeSrcPath)
exprTypeSrc = glob(exprTypeSrcPath)
pkg/discovery/types/type_set.gen.go
Generated
+1 -1
View File
@@ -6,7 +6,7 @@ package types
// the code is regenerated.
//
// Definitions file that controls how this file is generated:
// pkg/discovery/types.yaml
// pkg/discovery/types/types.yaml
type (
pkg/discovery/types/type_set.gen_test.go
Generated
+1 -1
View File
@@ -6,7 +6,7 @@ package types
// the code is regenerated.
//
// Definitions file that controls how this file is generated:
// pkg/discovery/types.yaml
// pkg/discovery/types/types.yaml
import (
"fmt"
pkg/label/types/type_set.gen.go
Generated
+2 -4
View File
@@ -6,7 +6,7 @@ package types
// the code is regenerated.
//
// Definitions file that controls how this file is generated:
// pkg/label/types.yaml
// pkg/label/types/types.yaml
type (
@@ -14,8 +14,6 @@ type (
//
// This type is auto-generated.
LabelSet []*Label
Map map[string]string
)
// Walk iterates through every slice item and calls w(Label) err
@@ -31,7 +29,7 @@ func (set LabelSet) Walk(w func(*Label) error) (err error) {
return
}
// LabelFilter iterates through every slice item, calls f(Label) (bool, err) and return filtered slice
// Filter iterates through every slice item, calls f(Label) (bool, err) and return filtered slice
//
// This function is auto-generated.
func (set LabelSet) Filter(f func(*Label) (bool, error)) (out LabelSet, err error) {
pkg/label/types/type_set.gen_test.go
Generated
+1 -1
View File
@@ -6,7 +6,7 @@ package types
// the code is regenerated.
//
// Definitions file that controls how this file is generated:
// pkg/label/types.yaml
// pkg/label/types/types.yaml
import (
"fmt"
tests/dal/main_test.go
+1 -1
View File
@@ -351,7 +351,7 @@ func createRecordFrom(ctx context.Context, t *testing.T, suite, name string, nam
record.NamespaceID = namespaceID
record.ModuleID = moduleID
record, err := composeService.DefaultRecord.Create(ctx, record)
record, _, err := composeService.DefaultRecord.Create(ctx, record)
require.NoError(t, err)
return record