diff --git a/compose/automation/records_handler.go b/compose/automation/records_handler.go index c5368ce7d..a5d0b86a0 100644 --- a/compose/automation/records_handler.go +++ b/compose/automation/records_handler.go @@ -13,12 +13,12 @@ import ( type ( recordService interface { - FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, error) + FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, *types.RecordValueErrorSet, error) Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) - Create(ctx context.Context, record *types.Record) (*types.Record, error) - Update(ctx context.Context, record *types.Record) (*types.Record, error) - Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, error) + Create(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error) + Update(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error) + Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, *types.RecordValueErrorSet, error) Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (out any, err error) Validate(ctx context.Context, rec *types.Record) error @@ -273,13 +273,13 @@ func (h recordsHandler) new(ctx context.Context, args *recordsNewArgs) (*records func (h recordsHandler) create(ctx context.Context, args *recordsCreateArgs) (results *recordsCreateResults, err error) { results = &recordsCreateResults{} - results.Record, err = h.rec.Create(ctx, args.Record) + results.Record, _, err = h.rec.Create(ctx, args.Record) return } func (h recordsHandler) update(ctx context.Context, args *recordsUpdateArgs) (results *recordsUpdateResults, err error) { results = &recordsUpdateResults{} - results.Record, err = h.rec.Update(ctx, args.Record) + results.Record, _, err = h.rec.Update(ctx, args.Record) return } @@ -322,7 +322,8 @@ func (h recordsHandler) lookupRecord(ctx context.Context, args recordLookup) (re return } - return h.rec.FindByID(ctx, namespace.ID, module.ID, recordID) + record, _, err = h.rec.FindByID(ctx, namespace.ID, module.ID, recordID) + return } func (h recordsHandler) loadCombo(ctx context.Context, args interface{}) (namespace *types.Namespace, module *types.Module, err error) { diff --git a/compose/rest/record.go b/compose/rest/record.go index c2c6434d8..44e79134c 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -33,7 +33,8 @@ type ( recordPayload struct { *types.Record - Records types.RecordSet `json:"records,omitempty"` + Records types.RecordSet `json:"records,omitempty"` + RecordValueErrors *types.RecordValueErrorSet `json:"valueErrors"` CanManageOwnerOnRecord bool `json:"canManageOwnerOnRecord"` CanUpdateRecord bool `json:"canUpdateRecord"` @@ -147,14 +148,14 @@ func (ctrl *Record) Read(ctx context.Context, r *request.RecordRead) (interface{ return nil, err } - record, err := ctrl.record.FindByID(ctx, r.NamespaceID, r.ModuleID, r.RecordID) + record, dd, err := ctrl.record.FindByID(ctx, r.NamespaceID, r.ModuleID, r.RecordID) // Temp workaround until we do proper by-module filtering for record findByID if record != nil && record.ModuleID != r.ModuleID { return nil, store.ErrNotFound } - return ctrl.makePayload(ctx, m, record, err) + return ctrl.makePayload(ctx, m, record, dd, err) } func (ctrl *Record) Create(ctx context.Context, r *request.RecordCreate) (interface{}, error) { @@ -200,12 +201,12 @@ func (ctrl *Record) Create(ctx context.Context, r *request.RecordCreate) (interf } oo = append(oo, oob...) - rr, err := ctrl.record.Bulk(ctx, oo...) + rr, dd, err := ctrl.record.Bulk(ctx, oo...) if rve := types.IsRecordValueErrorSet(err); rve != nil { return ctrl.handleValidationError(rve), nil } - return ctrl.makeBulkPayload(ctx, m, err, rr...) + return ctrl.makeBulkPayload(ctx, m, dd, err, rr...) } func (ctrl *Record) Update(ctx context.Context, r *request.RecordUpdate) (interface{}, error) { @@ -252,13 +253,12 @@ func (ctrl *Record) Update(ctx context.Context, r *request.RecordUpdate) (interf } oo = append(oo, oob...) - rr, err := ctrl.record.Bulk(ctx, oo...) - + rr, dd, err := ctrl.record.Bulk(ctx, oo...) if rve := types.IsRecordValueErrorSet(err); rve != nil { return ctrl.handleValidationError(rve), nil } - return ctrl.makeBulkPayload(ctx, m, err, rr...) + return ctrl.makeBulkPayload(ctx, m, dd, err, rr...) } func (ctrl *Record) Delete(ctx context.Context, r *request.RecordDelete) (interface{}, error) { @@ -550,7 +550,7 @@ func (ctrl *Record) TriggerScript(ctx context.Context, r *request.RecordTriggerS module, record, err := ctrl.record.TriggerScript(ctx, r.NamespaceID, r.ModuleID, r.RecordID, r.Values, r.Script) // Script can return modified record and we'll pass it on to the caller - return ctrl.makePayload(ctx, module, record, err) + return ctrl.makePayload(ctx, module, record, nil, err) } func (ctrl *Record) TriggerScriptOnList(ctx context.Context, r *request.RecordTriggerScriptOnList) (rsp interface{}, err error) { @@ -604,14 +604,15 @@ func (ctrl *Record) Revisions(ctx context.Context, r *request.RecordRevisions) ( }, err } -func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err error, rr ...*types.Record) (*recordPayload, error) { +func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, dd *types.RecordValueErrorSet, err error, rr ...*types.Record) (*recordPayload, error) { if err != nil || rr == nil { return nil, err } return &recordPayload{ - Record: rr[0], - Records: rr[1:], + Record: rr[0], + Records: rr[1:], + RecordValueErrors: dd, CanManageOwnerOnRecord: ctrl.ac.CanManageOwnerOnRecord(ctx, rr[0]), CanUpdateRecord: ctrl.ac.CanUpdateRecord(ctx, rr[0]), @@ -621,13 +622,14 @@ func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err err }, nil } -func (ctrl Record) makePayload(ctx context.Context, m *types.Module, r *types.Record, err error) (*recordPayload, error) { +func (ctrl Record) makePayload(ctx context.Context, m *types.Module, r *types.Record, dd *types.RecordValueErrorSet, err error) (*recordPayload, error) { if err != nil || r == nil { return nil, err } return &recordPayload{ - Record: r, + Record: r, + RecordValueErrors: dd, CanGrant: ctrl.ac.CanGrant(ctx), @@ -647,7 +649,7 @@ func (ctrl Record) makeFilterPayload(ctx context.Context, m *types.Module, rr ty modp := &recordSetPayload{Filter: f, Set: make([]*recordPayload, len(rr))} for i := range rr { - modp.Set[i], _ = ctrl.makePayload(ctx, m, rr[i], nil) + modp.Set[i], _ = ctrl.makePayload(ctx, m, rr[i], nil, nil) } return modp, nil diff --git a/compose/service/record.go b/compose/service/record.go index fc263560a..59db99c67 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -51,9 +51,10 @@ type ( revisions *recordRevisions - formatter recordValuesFormatter - sanitizer recordValuesSanitizer - validator recordValuesValidator + formatter recordValuesFormatter + sanitizer recordValuesSanitizer + validator recordValuesValidator + dupDetector recordValuesDupDetector } recordValuesFormatter interface { @@ -72,6 +73,10 @@ type ( UserRefChecker(fn values.ReferenceChecker) } + recordValuesDupDetector interface { + CheckDuplication(context.Context, types.DeDupRuleSet, types.Record, types.RecordSet) (*types.RecordValueErrorSet, error) + } + recordValueAccessController interface { CanReadRecordValueOnModuleField(context.Context, *types.ModuleField) bool CanUpdateRecordValueOnModuleField(context.Context, *types.ModuleField) bool @@ -105,7 +110,7 @@ type ( } RecordService interface { - FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, error) + FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, *types.RecordValueErrorSet, error) Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (any, error) Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) @@ -114,9 +119,9 @@ type ( RecordExport(context.Context, types.RecordFilter) error RecordImport(context.Context, error) error - Create(ctx context.Context, record *types.Record) (*types.Record, error) - Update(ctx context.Context, record *types.Record) (*types.Record, error) - Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, error) + Create(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error) + Update(ctx context.Context, record *types.Record) (*types.Record, *types.RecordValueErrorSet, error) + Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (types.RecordSet, *types.RecordValueErrorSet, error) Validate(ctx context.Context, rec *types.Record) error @@ -185,8 +190,9 @@ func Record() *record { revisions: &recordRevisions{revisions.Service(dal.Service())}, - formatter: values.Formatter(), - sanitizer: values.Sanitizer(), + formatter: values.Formatter(), + sanitizer: values.Sanitizer(), + dupDetector: types.DeDup(), } svc.validator = defaultValidator(svc) @@ -212,7 +218,7 @@ func defaultValidator(svc RecordService) recordValuesValidator { return false, nil } - r, err := svc.FindByID(ctx, f.NamespaceID, f.ModuleID, v.Ref) + r, _, err := svc.FindByID(ctx, f.NamespaceID, f.ModuleID, v.Ref) return r != nil, err }) @@ -234,7 +240,7 @@ func defaultValidator(svc RecordService) recordValuesValidator { } // lookup fn() orchestrates record lookup, namespace preload and check -func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, lookup func(*types.Module, *recordActionProps) (*types.Record, error)) (r *types.Record, err error) { +func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, lookup func(*types.Module, *recordActionProps) (*types.Record, error)) (r *types.Record, dd *types.RecordValueErrorSet, err error) { var ( ns *types.Namespace m *types.Module @@ -266,13 +272,15 @@ func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, look r.SetModule(m) r.Values = svc.sanitizer.RunXSS(m, r.Values) + dd, err = svc.DupDetection(ctx, m, r) + return nil }() - return r, svc.recordAction(ctx, aProps, RecordActionLookup, err) + return r, dd, svc.recordAction(ctx, aProps, RecordActionLookup, err) } -func (svc record) FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (r *types.Record, err error) { +func (svc record) FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (r *types.Record, dd *types.RecordValueErrorSet, err error) { return svc.lookup(ctx, namespaceID, moduleID, func(m *types.Module, props *recordActionProps) (*types.Record, error) { props.record.ID = recordID @@ -509,7 +517,7 @@ func (svc record) RecordExport(ctx context.Context, f types.RecordFilter) (err e // Bulk handles provided set of bulk record operations. // It's able to create, update or delete records in a single transaction. -func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (rr types.RecordSet, err error) { +func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (rr types.RecordSet, dd *types.RecordValueErrorSet, err error) { var pr *types.Record err = func() error { @@ -563,11 +571,11 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r switch p.Operation { case types.OperationTypeCreate: action = RecordActionCreate - r, err = svc.create(ctx, r) + r, dd, err = svc.create(ctx, r) case types.OperationTypeUpdate: action = RecordActionUpdate - r, err = svc.update(ctx, r) + r, dd, err = svc.update(ctx, r) case types.OperationTypeDelete: action = RecordActionDelete @@ -614,20 +622,20 @@ func (svc record) Bulk(ctx context.Context, oo ...*types.RecordBulkOperation) (r if len(oo) == 1 { // was not really a bulk operation, and we already recorded the action // inside transaction loop - return rr, err + return rr, dd, err } else { // when doing bulk op (updating and/or creating more than one record at once), // we already log action for each operation // // to log the fact that the bulk op was done, we do one additional recording // without any props - return rr, svc.recordAction(ctx, &recordActionProps{}, RecordActionBulk, err) + return rr, dd, svc.recordAction(ctx, &recordActionProps{}, RecordActionBulk, err) } } // Raw create function that is responsible for value validation, event dispatching // and creation. -func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Record, err error) { +func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) { var ( aProps = &recordActionProps{record: new} invokerID = auth.GetIdentityFromContext(ctx).Identity() @@ -645,7 +653,7 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec aProps.setModule(m) if !svc.ac.CanCreateRecordOnModule(ctx, m) { - return nil, RecordErrNotAllowedToCreate() + return nil, dd, RecordErrNotAllowedToCreate() } if err = RecordValueSanitization(m, new.Values); err != nil { @@ -661,21 +669,26 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec { if rve = svc.procCreate(ctx, invokerID, m, new); !rve.IsValid() { - return nil, RecordErrValueInput().Wrap(rve) + return nil, dd, RecordErrValueInput().Wrap(rve) } if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeCreate(new, nil, m, ns, rve, nil)); err != nil { return } else if !rve.IsValid() { - return nil, RecordErrValueInput().Wrap(rve) + return nil, dd, RecordErrValueInput().Wrap(rve) } } new.Values = RecordValueDefaults(m, new.Values) + dd, err = svc.DupDetection(ctx, m, new) + if err != nil { + return + } + // Handle payload from automation scripts if rve = svc.procCreate(ctx, invokerID, m, new); !rve.IsValid() { - return nil, RecordErrValueInput().Wrap(rve) + return nil, dd, RecordErrValueInput().Wrap(rve) } aProps.setChanged(new) @@ -828,6 +841,7 @@ func CalcRecordOwner(current, new, invoker uint64) uint64 { return new } +// @todo: ?? this might be a good place for detection too func RecordValueUpdateOpCheck(ctx context.Context, ac recordValueAccessController, m *types.Module, vv types.RecordValueSet) *types.RecordValueErrorSet { rve := &types.RecordValueErrorSet{} if ac == nil { @@ -905,7 +919,7 @@ func RecordValueDefaults(m *types.Module, vv types.RecordValueSet) (out types.Re // Raw update function that is responsible for value validation, event dispatching // and update. -func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Record, err error) { +func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) { var ( aProps = &recordActionProps{record: upd} invokerID = auth.GetIdentityFromContext(ctx).Identity() @@ -916,7 +930,7 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec ) if upd.ID == 0 { - return nil, RecordErrInvalidID() + return nil, dd, RecordErrInvalidID() } ns, m, old, err = loadRecordCombo(ctx, svc.store, svc.dal, upd.NamespaceID, upd.ModuleID, upd.ID) @@ -929,12 +943,12 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec aProps.setRecord(old) if !svc.ac.CanUpdateRecord(ctx, old) { - return nil, RecordErrNotAllowedToUpdate() + return nil, dd, RecordErrNotAllowedToUpdate() } // Test if stale (update has an older version of data) if isStale(upd.UpdatedAt, old.UpdatedAt, old.CreatedAt) { - return nil, RecordErrStaleData() + return nil, dd, RecordErrStaleData() } if err = RecordValueSanitization(m, upd.Values); err != nil { @@ -949,10 +963,15 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec upd.SetModule(m) old.SetModule(m) + dd, err = svc.DupDetection(ctx, m, upd) + if err != nil { + return + } + { // Handle input payload if rve = svc.procUpdate(ctx, invokerID, m, upd, old); !rve.IsValid() { - return nil, RecordErrValueInput().Wrap(rve) + return nil, dd, RecordErrValueInput().Wrap(rve) } // Scripts can (besides simple error value) return complex record value error set @@ -963,13 +982,13 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeUpdate(upd, old, m, ns, rve, nil)); err != nil { return } else if !rve.IsValid() { - return nil, RecordErrValueInput().Wrap(rve) + return nil, dd, RecordErrValueInput().Wrap(rve) } } // Handle payload from automation scripts if rve = svc.procUpdate(ctx, invokerID, m, upd, old); !rve.IsValid() { - return nil, RecordErrValueInput().Wrap(rve) + return nil, dd, RecordErrValueInput().Wrap(rve) } err = store.Tx(ctx, svc.store, func(ctx context.Context, s store.Storer) error { @@ -990,7 +1009,7 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec }) if err != nil { - return nil, err + return nil, dd, err } // ensure module ref is set before running through records workflows and scripts @@ -1012,18 +1031,18 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec return } -func (svc record) Create(ctx context.Context, new *types.Record) (rec *types.Record, err error) { +func (svc record) Create(ctx context.Context, new *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) { var ( aProps = &recordActionProps{record: new} ) err = func() error { - rec, err = svc.create(ctx, new) + rec, dd, err = svc.create(ctx, new) aProps.setRecord(rec) return err }() - return rec, svc.recordAction(ctx, aProps, RecordActionCreate, err) + return rec, dd, svc.recordAction(ctx, aProps, RecordActionCreate, err) } // Runs value sanitization, sets values that should be used @@ -1079,18 +1098,18 @@ func (svc record) procCreate(ctx context.Context, invokerID uint64, m *types.Mod return rve } -func (svc record) Update(ctx context.Context, upd *types.Record) (rec *types.Record, err error) { +func (svc record) Update(ctx context.Context, upd *types.Record) (rec *types.Record, dd *types.RecordValueErrorSet, err error) { var ( aProps = &recordActionProps{record: upd} ) err = func() error { - rec, err = svc.update(ctx, upd) + rec, dd, err = svc.update(ctx, upd) aProps.setRecord(rec) return err }() - return rec, svc.recordAction(ctx, aProps, RecordActionUpdate, err) + return rec, dd, svc.recordAction(ctx, aProps, RecordActionUpdate, err) } // Runs value sanitization, copies values that should updated @@ -1628,6 +1647,49 @@ func (svc record) Iterator(ctx context.Context, f types.RecordFilter, fn eventbu return svc.recordAction(ctx, aProps, RecordActionIteratorInvoked, err) } +// DupDetection check for any duplicate records and returns error for strict duplication +func (svc record) DupDetection(ctx context.Context, m *types.Module, rec *types.Record) (out *types.RecordValueErrorSet, err error) { + if m == nil || rec == nil { + return + } + + // @todo: improve per records duplicate detection, + // since it is bit too extreme besides we do have bulk operations + var ( + records types.RecordSet + + rProps = &recordActionProps{} + config = m.Config.RecordDeDup + ) + + if config.Enabled { + records, _, err = svc.Find(ctx, types.RecordFilter{ + ModuleID: m.ID, + NamespaceID: m.NamespaceID, + }) + if err != nil { + return + } + + out, err = svc.dupDetector.CheckDuplication(ctx, config.Rules, *rec, records) + if err != nil { + return + } + + // @todo: improve error string with details + rProps.setValueErrors(out) + + // Error out if duplicate record exist + if (config.Strict && !out.IsValid()) || out.HasStrictErrors() { + return out, types.IsRecordValueErrorSet(out) + } else { + return out, nil + } + } + + return +} + func ComposeRecordFilterChecker(ctx context.Context, ac recordAccessController, m *types.Module) func(*types.Record) (bool, error) { return func(rec *types.Record) (bool, error) { // Setting module right before we do access control diff --git a/compose/service/record_test.go b/compose/service/record_test.go index 48a44e0e6..db2d03463 100644 --- a/compose/service/record_test.go +++ b/compose/service/record_test.go @@ -323,13 +323,13 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) { // security context w/ writer role ctx = auth.SetIdentityToContext(ctx, auth.Authenticated(u.ID, writerRole.ID, authRoleID)) - recChecked, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valChecked}) + recChecked, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valChecked}) verifyRecErrSet(t, err) req.NotNil(recChecked.Values.Get("bool", 0), "should be checked") req.Equal("1", recChecked.Values.Get("bool", 0).Value) - recUnchecked, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valUnchecked}) + recUnchecked, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: valUnchecked}) req.NoError(err) req.Nil(recUnchecked.Values.Get("bool", 0)) @@ -343,7 +343,7 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) { &types.RecordValue{Name: "string", Value: "abc"}, } - recChecked, err = svc.Update(ctx, recChecked) + recChecked, _, err = svc.Update(ctx, recChecked) req.NoError(err) req.NotNil(recChecked.Values.Get("bool", 0), "should still be checked") @@ -353,7 +353,7 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) { &types.RecordValue{Name: "string", Value: "abc"}, } - recUnchecked, err = svc.Update(ctx, recUnchecked) + recUnchecked, _, err = svc.Update(ctx, recUnchecked) req.NoError(err) req.Nil(recUnchecked.Values.Get("bool", 0), "should not be checked anymore") @@ -367,7 +367,7 @@ func TestRecord_boolFieldPermissionIssueKBR(t *testing.T) { &types.RecordValue{Name: "bool", Value: "1"}, } - recChecked, err = svc.Update(ctx, recChecked) + recChecked, _, err = svc.Update(ctx, recChecked) req.NoError(err) req.NotNil(recChecked.Values.Get("bool", 0), "should checked again") @@ -462,13 +462,13 @@ func TestRecord_defValueFieldPermissionIssue(t *testing.T) { ctx = auth.SetIdentityToContext(ctx, auth.Authenticated(user.ID, authRoleID)) - recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}}) + recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}}) verifyRecErrSet(t, err) req.Equal("", valueExtractor(recPartial, "writable", "readable")) t.Log("creating record with w/o editor role (must be able to crate & update record and modify both fields)") - recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{ + recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{ &types.RecordValue{Name: "writable", Value: "w"}, &types.RecordValue{Name: "readable", Value: "r"}, }}) @@ -480,7 +480,7 @@ func TestRecord_defValueFieldPermissionIssue(t *testing.T) { recPartial.Values = types.RecordValueSet{&types.RecordValue{Name: "writable", Value: "w2"}} - recPartial, err = svc.Update(ctx, recPartial) + recPartial, _, err = svc.Update(ctx, recPartial) verifyRecErrSet(t, err) req.Equal("", valueExtractor(recPartial, "writable", "readable")) } @@ -490,13 +490,13 @@ func TestRecord_defValueFieldPermissionIssue(t *testing.T) { ctx = auth.SetIdentityToContext(ctx, auth.Authenticated(user.ID, authRoleID, editorRole.ID)) - recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}}) + recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{}}) verifyRecErrSet(t, err) req.Equal("", valueExtractor(recPartial, "writable", "readable")) t.Log("creating record with editor role (must be able to crate & update record and modify both fields)") - recPartial, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{ + recPartial, _, err = svc.Create(ctx, &types.Record{ModuleID: mod.ID, NamespaceID: ns.ID, Values: types.RecordValueSet{ // this is the def. value set &types.RecordValue{Name: "writable", Value: "def-w"}, &types.RecordValue{Name: "readable", Value: "r"}, @@ -582,14 +582,14 @@ func TestRecord_refAccessControl(t *testing.T) { { t.Log("creating record on 1st module; should failed because we do not have permissions to create records") - _, err = svc.Create(ctx, mod1rec1) + _, _, err = svc.Create(ctx, mod1rec1) req.EqualError(err, "not allowed to create records") t.Logf("granting permissions to create records on this module") req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, mod1.RbacResource(), "record.create"))) t.Log("retry creating record on 1st module; should fail because we do not have permissions to update field") - _, err = svc.Create(ctx, mod1rec1) + _, _, err = svc.Create(ctx, mod1rec1) req.Error(err) req.True(types.IsRecordValueErrorSet(err).HasKind("updateDenied")) @@ -597,12 +597,12 @@ func TestRecord_refAccessControl(t *testing.T) { req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, mod1strField.RbacResource(), "record.value.update"))) t.Log("retry creating record on 1st module; should succeed") - mod1rec1, err = svc.Create(ctx, mod1rec1) + mod1rec1, _, err = svc.Create(ctx, mod1rec1) req.NoError(err) } { t.Log("can record be read") - _, err = svc.FindByID(ctx, mod1rec1.NamespaceID, mod1rec1.ModuleID, mod1rec1.ID) + _, _, err = svc.FindByID(ctx, mod1rec1.NamespaceID, mod1rec1.ModuleID, mod1rec1.ID) req.EqualError(err, "not allowed to read this record") } { @@ -610,7 +610,7 @@ func TestRecord_refAccessControl(t *testing.T) { mod2rec1.Values = mod2rec1.Values.Set(&types.RecordValue{Name: "ref", Value: fmt.Sprintf("%d", mod1rec1.ID)}) t.Log("create record on 2nd module with ref to record on the 1st module; must fail, no create perm") - _, err = svc.Create(ctx, mod2rec1) + _, _, err = svc.Create(ctx, mod2rec1) req.EqualError(err, "not allowed to create records") t.Log("grant record.create on namespace level") @@ -620,36 +620,36 @@ func TestRecord_refAccessControl(t *testing.T) { req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, types.ModuleFieldRbacResource(ns.ID, 0, 0), "record.value.update"))) t.Log("create record on 2nd module with ref to record on the 1st module; most fail, not allowed to read (referenced) mod1rec1") - _, err = svc.Create(ctx, mod2rec1) + _, _, err = svc.Create(ctx, mod2rec1) req.EqualError(err, "invalid record value input") t.Log("grant read on record") req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, mod1rec1.RbacResource(), "read"))) t.Log("create record on 2nd module with ref to record on the 1st module") - mod2rec1, err = svc.Create(ctx, mod2rec1) + mod2rec1, _, err = svc.Create(ctx, mod2rec1) verifyRecErrSet(t, err) } { t.Log("update record on 2nd module with unchanged values; must fail, no update permissions") - _, err = svc.Update(ctx, mod2rec1) + _, _, err = svc.Update(ctx, mod2rec1) req.EqualError(err, "not allowed to update this record") t.Log("grant update on namespace level") req.NoError(rbacService.Grant(ctx, rbac.AllowRule(testerRole.ID, types.RecordRbacResource(ns.ID, 0, 0), "update"))) t.Log("update record on 2nd module with unchanged values") - mod2rec1, err = svc.Update(ctx, mod2rec1) + mod2rec1, _, err = svc.Update(ctx, mod2rec1) verifyRecErrSet(t, err) t.Log("update record on 2nd module with unchanged values; unset record value") mod2rec1.Values = nil - mod2rec1, err = svc.Update(ctx, mod2rec1) + mod2rec1, _, err = svc.Update(ctx, mod2rec1) verifyRecErrSet(t, err) t.Log("link 2nd record to 1st one again") mod2rec1.Values = mod2rec1.Values.Set(&types.RecordValue{Name: "ref", Value: fmt.Sprintf("%d", mod1rec1.ID)}) - mod2rec1, err = svc.Update(ctx, mod2rec1) + mod2rec1, _, err = svc.Update(ctx, mod2rec1) verifyRecErrSet(t, err) } { @@ -658,7 +658,7 @@ func TestRecord_refAccessControl(t *testing.T) { t.Log("link 2nd record to 1st one again but w/o permissions; must work, value did not change") mod2rec1.Values = mod2rec1.Values.Set(&types.RecordValue{Name: "ref", Value: fmt.Sprintf("%d", mod1rec1.ID)}) - mod2rec1, err = svc.Update(ctx, mod2rec1) + mod2rec1, _, err = svc.Update(ctx, mod2rec1) verifyRecErrSet(t, err) } } diff --git a/compose/types/module.go b/compose/types/module.go index 359e1b301..e4e673484 100644 --- a/compose/types/module.go +++ b/compose/types/module.go @@ -6,6 +6,7 @@ import ( "time" discovery "github.com/cortezaproject/corteza-server/discovery/types" + "github.com/cortezaproject/corteza-server/pkg/sql" "github.com/jmoiron/sqlx/types" @@ -58,6 +59,9 @@ type ( Discovery discovery.ModuleMeta `json:"discovery"` RecordRevisions ModuleConfigRecordRevisions `json:"recordRevisions"` + + // RecordDeDup value duplicate detection settings + RecordDeDup ModuleConfigRecordDeDup `json:"recordDeDup"` } ModuleConfigDAL struct { @@ -88,6 +92,18 @@ type ( UsageDisclosure string `json:"usageDisclosure"` } + ModuleConfigRecordDeDup struct { + // enable or disable duplicate detection + Enabled bool `json:"enabled"` + + // strictly restrict record saving + // otherwise show a warning with list of duplicated records + Strict bool `json:"strict"` + + // list of duplicate detection rules applied to module's fields + Rules DeDupRuleSet `json:"rules,omitempty"` + } + ModuleFilter struct { ModuleID []uint64 `json:"moduleID"` NamespaceID uint64 `json:"namespaceID,string"` diff --git a/compose/types/record.go b/compose/types/record.go index 94bb5a9af..59592c391 100644 --- a/compose/types/record.go +++ b/compose/types/record.go @@ -380,3 +380,29 @@ func (set RecordBulkSet) ToBulkOperations(dftModule uint64, dftNamespace uint64) return } + +// GetValuesByName filters values for records by names +func (set RecordSet) GetValuesByName(names ...string) (out RecordValueSet) { + nameMap := make(map[string]bool) + for _, n := range names { + if len(n) > 0 { + nameMap[n] = true + } + } + + err := set.Walk(func(rec *Record) error { + _ = rec.Values.Walk(func(val *RecordValue) error { + if val != nil && nameMap[val.Name] { + val.RecordID = rec.ID + out = append(out, val) + } + return nil + }) + return nil + }) + if err != nil { + return + } + + return +} diff --git a/compose/types/record_detect_duplicates.go b/compose/types/record_detect_duplicates.go new file mode 100644 index 000000000..6948fcc4c --- /dev/null +++ b/compose/types/record_detect_duplicates.go @@ -0,0 +1,167 @@ +package types + +import ( + "context" + "fmt" + "github.com/cortezaproject/corteza-server/pkg/locale" + "github.com/spf13/cast" + "strings" +) + +type ( + deDup struct { + ls localeService + } + + localeService interface { + T(ctx context.Context, ns, key string, rr ...string) string + } + + DeDupRule struct { + Name DeDupRuleName `json:"name"` + Strict bool `json:"strict"` + Attributes []string `json:"attributes"` + } + + // DeDupRuleName represent the identifier for duplicate detection rule + DeDupRuleName string + + // DeDupIssueKind based on strict mode rule or duplication config + DeDupIssueKind string +) + +const ( + caseSensitive DeDupRuleName = "case-sensitive" + + dupWarning DeDupIssueKind = "duplication_warning" + dupError DeDupIssueKind = "duplication_error" +) + +func DeDup() *deDup { + return &deDup{ + ls: locale.Global(), + } +} + +func (d deDup) CheckDuplication(ctx context.Context, rules DeDupRuleSet, rec Record, rr RecordSet) (out *RecordValueErrorSet, err error) { + out = &RecordValueErrorSet{} + err = rules.Walk(func(rule *DeDupRule) error { + if rule.HasAttributes() { + values := rr.GetValuesByName(distinct(rule.Attributes)...) + + set := rule.validateValue(ctx, d.ls, rec, values) + + if !set.IsValid() { + out.Push(set.Set...) + } + } + return nil + }) + if err != nil { + return + } + + return +} + +func (rule DeDupIssueKind) String() string { + return string(rule) +} + +func (rule DeDupRule) HasAttributes() bool { + return len(rule.Attributes) > 0 +} + +func (rule DeDupRule) IsStrict() bool { + return rule.Strict +} + +func (rule DeDupRule) IssueKind() string { + out := dupWarning + if rule.Strict { + out = dupError + } + + return out.String() +} + +func (rule DeDupRule) IssueMessage() (out string) { + return "record-field.errors.duplicateValue" +} + +func (rule DeDupRule) String() string { + return fmt.Sprintf("%s duplicate detection on `%s` field", rule.Name, strings.Join(rule.Attributes, ", ")) +} + +// validateValue will check duplicate detection based on rules name +func (rule DeDupRule) validateValue(ctx context.Context, ls localeService, rec Record, vv RecordValueSet) (out *RecordValueErrorSet) { + switch rule.Name { + case caseSensitive: + return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv) + default: + return rule.checkCaseSensitiveDuplication(ctx, ls, rec, vv) + } +} + +func (rule DeDupRule) checkCaseSensitiveDuplication(ctx context.Context, ls localeService, rec Record, vv RecordValueSet) (out *RecordValueErrorSet) { + out = &RecordValueErrorSet{} + recVal := rec.Values + + _ = recVal.Walk(func(newV *RecordValue) error { + _ = vv.Walk(func(v *RecordValue) error { + if v.RecordID != rec.ID { + if toLower(v.Value) == toLower(newV.Value) { + out.Push(RecordValueError{ + Kind: rule.IssueKind(), + Message: ls.T(ctx, "compose", rule.IssueMessage()), + Meta: map[string]interface{}{ + "dupValueField": newV.Name, + "recordID": cast.ToString(v.RecordID), + "id": cast.ToString(v.RecordID), + "field": v.Name, + "value": v.Value, + "rule": rule.String(), + }, + }) + } + } + return nil + }) + return nil + }) + return +} + +func (v *RecordValueErrorSet) HasStrictErrors() bool { + return v.HasKind(dupError.String()) +} + +// CaseSensitiveDuplicationRule prepares the case-sensitive duplicate detection rule +func CaseSensitiveDuplicationRule(strict bool, identifiers ...string) DeDupRule { + return makeDuplicationRule(caseSensitive, strict, identifiers...) +} + +// makeDuplicationRule prepares duplication detection rules +func makeDuplicationRule(name DeDupRuleName, strict bool, attributes ...string) DeDupRule { + return DeDupRule{ + Name: name, + Strict: strict, + Attributes: attributes, + } +} + +// distinct only list the different (distinct) values +func distinct(input []string) (out []string) { + keys := make(map[string]bool) + for _, val := range input { + if _, ok := keys[val]; !ok { + keys[val] = true + out = append(out, val) + } + } + return +} + +func toLower(s string) string { + return strings.ToLower(s) +} diff --git a/compose/types/type_set.gen.go b/compose/types/type_set.gen.go index d29ff9490..580bc1ed4 100644 --- a/compose/types/type_set.gen.go +++ b/compose/types/type_set.gen.go @@ -20,6 +20,11 @@ type ( // This type is auto-generated. ChartSet []*Chart + // DeDupRuleSet slice of DeDupRule + // + // This type is auto-generated. + DeDupRuleSet []*DeDupRule + // ModuleSet slice of Module // // This type is auto-generated. @@ -168,6 +173,36 @@ func (set ChartSet) IDs() (IDs []uint64) { return } +// Walk iterates through every slice item and calls w(DeDupRule) err +// +// This function is auto-generated. +func (set DeDupRuleSet) Walk(w func(*DeDupRule) error) (err error) { + for i := range set { + if err = w(set[i]); err != nil { + return + } + } + + return +} + +// Filter iterates through every slice item, calls f(DeDupRule) (bool, err) and return filtered slice +// +// This function is auto-generated. +func (set DeDupRuleSet) Filter(f func(*DeDupRule) (bool, error)) (out DeDupRuleSet, err error) { + var ok bool + out = DeDupRuleSet{} + for i := range set { + if ok, err = f(set[i]); err != nil { + return + } else if ok { + out = append(out, set[i]) + } + } + + return +} + // Walk iterates through every slice item and calls w(Module) err // // This function is auto-generated. diff --git a/compose/types/type_set.gen_test.go b/compose/types/type_set.gen_test.go index 6b38e4918..219d0de47 100644 --- a/compose/types/type_set.gen_test.go +++ b/compose/types/type_set.gen_test.go @@ -194,6 +194,62 @@ func TestChartSetIDs(t *testing.T) { } } +func TestDeDupRuleSetWalk(t *testing.T) { + var ( + value = make(DeDupRuleSet, 3) + req = require.New(t) + ) + + // check walk with no errors + { + err := value.Walk(func(*DeDupRule) error { + return nil + }) + req.NoError(err) + } + + // check walk with error + req.Error(value.Walk(func(*DeDupRule) error { return fmt.Errorf("walk error") })) +} + +func TestDeDupRuleSetFilter(t *testing.T) { + var ( + value = make(DeDupRuleSet, 3) + req = require.New(t) + ) + + // filter nothing + { + set, err := value.Filter(func(*DeDupRule) (bool, error) { + return true, nil + }) + req.NoError(err) + req.Equal(len(set), len(value)) + } + + // filter one item + { + found := false + set, err := value.Filter(func(*DeDupRule) (bool, error) { + if !found { + found = true + return found, nil + } + return false, nil + }) + req.NoError(err) + req.Len(set, 1) + } + + // filter error + { + _, err := value.Filter(func(*DeDupRule) (bool, error) { + return false, fmt.Errorf("filter error") + }) + req.Error(err) + } +} + func TestModuleSetWalk(t *testing.T) { var ( value = make(ModuleSet, 3) diff --git a/compose/types/types.yaml b/compose/types/types.yaml index cb9e3561b..98e5808a0 100644 --- a/compose/types/types.yaml +++ b/compose/types/types.yaml @@ -15,4 +15,5 @@ types: noIdField: true PrivacyModule: noIdField: true - + DeDupRule: + noIdField: true diff --git a/federation/service/processer_data_test.go b/federation/service/processer_data_test.go index 2cce99e6b..18691fed2 100644 --- a/federation/service/processer_data_test.go +++ b/federation/service/processer_data_test.go @@ -198,8 +198,8 @@ func TestProcesserData_persist(t *testing.T) { } // create success -func (s testRecordServicePersistSuccess) Create(_ context.Context, record *ct.Record) (*ct.Record, error) { - return nil, nil +func (s testRecordServicePersistSuccess) Create(_ context.Context, record *ct.Record) (*ct.Record, *ct.RecordValueErrorSet, error) { + return nil, nil, nil } func (s testRecordServicePersistSuccess) Find(_ context.Context, filter ct.RecordFilter) (ct.RecordSet, ct.RecordFilter, error) { @@ -207,8 +207,8 @@ func (s testRecordServicePersistSuccess) Find(_ context.Context, filter ct.Recor } // update success -func (s testRecordServiceUpdateSuccess) Update(_ context.Context, record *ct.Record) (*ct.Record, error) { - return nil, nil +func (s testRecordServiceUpdateSuccess) Update(_ context.Context, record *ct.Record) (*ct.Record, *ct.RecordValueErrorSet, error) { + return nil, nil, nil } func (s testRecordServiceUpdateSuccess) Find(_ context.Context, filter ct.RecordFilter) (ct.RecordSet, ct.RecordFilter, error) { @@ -225,6 +225,6 @@ func (s testRecordServiceDeleteSuccess) Find(_ context.Context, filter ct.Record } // create error -func (s testRecordServicePersistError) Create(_ context.Context, record *ct.Record) (*ct.Record, error) { - return nil, errors.New("mocked error") +func (s testRecordServicePersistError) Create(_ context.Context, record *ct.Record) (*ct.Record, *ct.RecordValueErrorSet, error) { + return nil, nil, errors.New("mocked error") } diff --git a/federation/service/sync.go b/federation/service/sync.go index adc7bc02f..96b3c21fb 100644 --- a/federation/service/sync.go +++ b/federation/service/sync.go @@ -79,13 +79,15 @@ func (s *Sync) FetchUrl(ctx context.Context, url string) (io.Reader, error) { } // CreateRecord wraps the compose Record service Create -func (s *Sync) CreateRecord(ctx context.Context, rec *ct.Record) (*ct.Record, error) { - return s.composeRecordService.Create(ctx, rec) +func (s *Sync) CreateRecord(ctx context.Context, rec *ct.Record) (out *ct.Record, err error) { + out, _, err = s.composeRecordService.Create(ctx, rec) + return } // UpdateRecord wraps the compose Record service Update -func (s *Sync) UpdateRecord(ctx context.Context, rec *ct.Record) (*ct.Record, error) { - return s.composeRecordService.Update(ctx, rec) +func (s *Sync) UpdateRecord(ctx context.Context, rec *ct.Record) (out *ct.Record, err error) { + out, _, err = s.composeRecordService.Update(ctx, rec) + return } // DeleteRecord wraps the compose Record service Update diff --git a/pkg/codegen/codegen.go b/pkg/codegen/codegen.go index 63bb2e4d5..863d98ac0 100644 --- a/pkg/codegen/codegen.go +++ b/pkg/codegen/codegen.go @@ -43,9 +43,10 @@ func Proc() { // workaround because // filepath.Join merges "*","*" into "**" instead of "*/*" - typeSrcPath = filepath.Join("*"+string(filepath.Separator)+"*", "types.yaml") - typeSrc []string - typeDefs []*typesDef + pkgTypeSrcPath = filepath.Join("*"+string(filepath.Separator)+"*"+string(filepath.Separator)+"*", "types.yaml") + typeSrcPath = filepath.Join("*"+string(filepath.Separator)+"*", "types.yaml") + typeSrc []string + typeDefs []*typesDef // workaround because // filepath.Join merges "*","*" into "**" instead of "*/*" @@ -144,6 +145,7 @@ func Proc() { output("loaded %d event definitions from %s\n", len(eventSrc), eventSrcPath) typeSrc = glob(typeSrcPath) + typeSrc = append(typeSrc, glob(pkgTypeSrcPath)...) output("loaded %d type definitions from %s\n", len(typeSrc), typeSrcPath) exprTypeSrc = glob(exprTypeSrcPath) diff --git a/pkg/discovery/types/type_set.gen.go b/pkg/discovery/types/type_set.gen.go index 8b0908c24..ea3af5693 100644 --- a/pkg/discovery/types/type_set.gen.go +++ b/pkg/discovery/types/type_set.gen.go @@ -6,7 +6,7 @@ package types // the code is regenerated. // // Definitions file that controls how this file is generated: -// pkg/discovery/types.yaml +// pkg/discovery/types/types.yaml type ( diff --git a/pkg/discovery/types/type_set.gen_test.go b/pkg/discovery/types/type_set.gen_test.go index af2dbc434..c2203db7c 100644 --- a/pkg/discovery/types/type_set.gen_test.go +++ b/pkg/discovery/types/type_set.gen_test.go @@ -6,7 +6,7 @@ package types // the code is regenerated. // // Definitions file that controls how this file is generated: -// pkg/discovery/types.yaml +// pkg/discovery/types/types.yaml import ( "fmt" diff --git a/pkg/label/types/type_set.gen.go b/pkg/label/types/type_set.gen.go index 842e58bba..ab446e02e 100644 --- a/pkg/label/types/type_set.gen.go +++ b/pkg/label/types/type_set.gen.go @@ -6,7 +6,7 @@ package types // the code is regenerated. // // Definitions file that controls how this file is generated: -// pkg/label/types.yaml +// pkg/label/types/types.yaml type ( @@ -14,8 +14,6 @@ type ( // // This type is auto-generated. LabelSet []*Label - - Map map[string]string ) // Walk iterates through every slice item and calls w(Label) err @@ -31,7 +29,7 @@ func (set LabelSet) Walk(w func(*Label) error) (err error) { return } -// LabelFilter iterates through every slice item, calls f(Label) (bool, err) and return filtered slice +// Filter iterates through every slice item, calls f(Label) (bool, err) and return filtered slice // // This function is auto-generated. func (set LabelSet) Filter(f func(*Label) (bool, error)) (out LabelSet, err error) { diff --git a/pkg/label/types/type_set.gen_test.go b/pkg/label/types/type_set.gen_test.go index 5d240181f..11e6cd294 100644 --- a/pkg/label/types/type_set.gen_test.go +++ b/pkg/label/types/type_set.gen_test.go @@ -6,7 +6,7 @@ package types // the code is regenerated. // // Definitions file that controls how this file is generated: -// pkg/label/types.yaml +// pkg/label/types/types.yaml import ( "fmt" diff --git a/tests/dal/main_test.go b/tests/dal/main_test.go index 492abc9f9..7b73802aa 100644 --- a/tests/dal/main_test.go +++ b/tests/dal/main_test.go @@ -351,7 +351,7 @@ func createRecordFrom(ctx context.Context, t *testing.T, suite, name string, nam record.NamespaceID = namespaceID record.ModuleID = moduleID - record, err := composeService.DefaultRecord.Create(ctx, record) + record, _, err := composeService.DefaultRecord.Create(ctx, record) require.NoError(t, err) return record