DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Add support for revisions on compose records

Browse Source
This commit is contained in:
Denis Arh
2022-08-01 08:55:46 +02:00
parent 39bf9b75ea
commit c9442260c5
22 changed files with 763 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/record.cue
+1
View File
@@ -43,6 +43,7 @@ record: schema.#Resource & {
"update": {}
"delete": {}
"owner.manage": {}
"revisions.search": {}
}
}
}
compose/rest.yaml
+7
View File
@@ -922,6 +922,13 @@ endpoints:
type: map[string]interface{}
parser: parseMapStringInterface
title: Arguments to pass to the script
- name: revisions
method: GET
title: List record revisions
path: "/{recordID}/revisions"
parameters:
path:
- { type: uint64, name: recordID, required: true, title: ID }
- title: Data Privacy
entrypoint: dataPrivacy
compose/rest/handlers/record.go
+20
View File
@@ -34,6 +34,7 @@ type (
Upload(context.Context, *request.RecordUpload) (interface{}, error)
TriggerScript(context.Context, *request.RecordTriggerScript) (interface{}, error)
TriggerScriptOnList(context.Context, *request.RecordTriggerScriptOnList) (interface{}, error)
Revisions(context.Context, *request.RecordRevisions) (interface{}, error)
}
// HTTP API interface
@@ -53,6 +54,7 @@ type (
Upload func(http.ResponseWriter, *http.Request)
TriggerScript func(http.ResponseWriter, *http.Request)
TriggerScriptOnList func(http.ResponseWriter, *http.Request)
Revisions func(http.ResponseWriter, *http.Request)
}
)
@@ -296,6 +298,23 @@ func NewRecord(h RecordAPI) *Record {
return
}
api.Send(w, r, value)
},
Revisions: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewRecordRevisions()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.
Revisions(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
}
api.Send(w, r, value)
},
}
@@ -319,5 +338,6 @@ func (h Record) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http
r.Post("/namespace/{namespaceID}/module/{moduleID}/record/attachment", h.Upload)
r.Post("/namespace/{namespaceID}/module/{moduleID}/record/{recordID}/trigger", h.TriggerScript)
r.Post("/namespace/{namespaceID}/module/{moduleID}/record/trigger", h.TriggerScriptOnList)
r.Get("/namespace/{namespaceID}/module/{moduleID}/record/{recordID}/revisions", h.Revisions)
})
}
compose/rest/record.go
+35 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/revisions"
"io"
"math"
"net/http"
@@ -37,7 +38,9 @@ type (
CanUpdateRecord bool `json:"canUpdateRecord"`
CanReadRecord bool `json:"canReadRecord"`
CanDeleteRecord bool `json:"canDeleteRecord"`
CanGrant bool `json:"canGrant"`
CanSearchRevisions bool `json:"canSearchRevisions"`
CanGrant bool `json:"canGrant"`
}
recordSetPayload struct {
@@ -61,6 +64,7 @@ type (
CanReadRecord(context.Context, *types.Record) bool
CanDeleteRecord(context.Context, *types.Record) bool
CanManageOwnerOnRecord(context.Context, *types.Record) bool
CanSearchRevisionsOnRecord(context.Context, *types.Record) bool
}
)
@@ -567,6 +571,34 @@ func (ctrl *Record) TriggerScriptOnList(ctx context.Context, r *request.RecordTr
return api.OK(), err
}
func (ctrl *Record) Revisions(ctx context.Context, r *request.RecordRevisions) (interface{}, error) {
var (
makeRev = func() dal.ValueSetter { return &revisions.Revision{} }
)
iter, err := ctrl.record.SearchRevisions(ctx, r.NamespaceID, r.ModuleID, r.RecordID)
if err != nil {
return nil, err
}
return func(w http.ResponseWriter, req *http.Request) {
if _, err = w.Write([]byte(`{"set":[`)); err != nil {
return
}
err = dal.IteratorEncodeJSON(ctx, w, iter, makeRev)
if err != nil {
return
}
if _, err = w.Write([]byte(`]}`)); err != nil {
return
}
return
}, err
}
func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err error, rr ...*types.Record) (*recordPayload, error) {
if err != nil || rr == nil {
return nil, err
@@ -580,6 +612,7 @@ func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err err
CanUpdateRecord: ctrl.ac.CanUpdateRecord(ctx, rr[0]),
CanReadRecord: ctrl.ac.CanReadRecord(ctx, rr[0]),
CanDeleteRecord: ctrl.ac.CanDeleteRecord(ctx, rr[0]),
CanSearchRevisions: ctrl.ac.CanSearchRevisionsOnRecord(ctx, rr[0]),
}, nil
}
@@ -597,6 +630,7 @@ func (ctrl Record) makePayload(ctx context.Context, m *types.Module, r *types.Re
CanUpdateRecord: ctrl.ac.CanUpdateRecord(ctx, r),
CanReadRecord: ctrl.ac.CanReadRecord(ctx, r),
CanDeleteRecord: ctrl.ac.CanDeleteRecord(ctx, r),
CanSearchRevisions: ctrl.ac.CanSearchRevisionsOnRecord(ctx, r),
}, nil
}
compose/rest/request/record.go
+76
View File
@@ -434,6 +434,23 @@ type (
// Arguments to pass to the script
Args map[string]interface{}
}
RecordRevisions struct {
// NamespaceID PATH parameter
//
// Namespace ID
NamespaceID uint64 `json:",string"`
// ModuleID PATH parameter
//
// Module ID
ModuleID uint64 `json:",string"`
// RecordID PATH parameter
//
// ID
RecordID uint64 `json:",string"`
}
)
// NewRecordReport request
@@ -2085,3 +2102,62 @@ func (r *RecordTriggerScriptOnList) Fill(req *http.Request) (err error) {
return err
}
// NewRecordRevisions request
func NewRecordRevisions() *RecordRevisions {
return &RecordRevisions{}
}
// Auditable returns all auditable/loggable parameters
func (r RecordRevisions) Auditable() map[string]interface{} {
return map[string]interface{}{
"namespaceID": r.NamespaceID,
"moduleID": r.ModuleID,
"recordID": r.RecordID,
}
}
// Auditable returns all auditable/loggable parameters
func (r RecordRevisions) GetNamespaceID() uint64 {
return r.NamespaceID
}
// Auditable returns all auditable/loggable parameters
func (r RecordRevisions) GetModuleID() uint64 {
return r.ModuleID
}
// Auditable returns all auditable/loggable parameters
func (r RecordRevisions) GetRecordID() uint64 {
return r.RecordID
}
// Fill processes request and fills internal variables
func (r *RecordRevisions) Fill(req *http.Request) (err error) {
{
var val string
// path params
val = chi.URLParam(req, "namespaceID")
r.NamespaceID, err = payload.ParseUint64(val), nil
if err != nil {
return err
}
val = chi.URLParam(req, "moduleID")
r.ModuleID, err = payload.ParseUint64(val), nil
if err != nil {
return err
}
val = chi.URLParam(req, "recordID")
r.RecordID, err = payload.ParseUint64(val), nil
if err != nil {
return err
}
}
return err
}
compose/service/access_control.gen.go
Generated
+17 -4
View File
@@ -291,6 +291,11 @@ func (svc accessControl) List() (out []map[string]string) {
"any": types.RecordRbacResource(0, 0, 0),
"op": "owner.manage",
},
{
"type": types.RecordResourceType,
"any": types.RecordRbacResource(0, 0, 0),
"op": "revisions.search",
},
{
"type": types.ComponentResourceType,
"any": types.ComponentRbacResource(),
@@ -609,6 +614,13 @@ func (svc accessControl) CanManageOwnerOnRecord(ctx context.Context, r *types.Re
return svc.can(ctx, "owner.manage", r)
}
// CanSearchRevisionsOnRecord checks if current user can revisions.search
//
// This function is auto-generated
func (svc accessControl) CanSearchRevisionsOnRecord(ctx context.Context, r *types.Record) bool {
return svc.can(ctx, "revisions.search", r)
}
// CanGrant checks if current user can manage compose permissions
//
// This function is auto-generated
@@ -790,10 +802,11 @@ func rbacResourceOperations(r string) map[string]bool {
}
case types.RecordResourceType:
return map[string]bool{
"read": true,
"update": true,
"delete": true,
"owner.manage": true,
"read": true,
"update": true,
"delete": true,
"owner.manage": true,
"revisions.search": true,
}
case types.ComponentResourceType:
return map[string]bool{
compose/service/module.go
+57 -14
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/revisions"
"reflect"
"sort"
"strconv"
@@ -79,8 +80,8 @@ type (
SearchModelIssues(connectionID, ID uint64) []error
}
dalIdentFormatter interface {
Format(ctx context.Context, template string) (out string, ok bool)
identFormatter interface {
Format(context.Context, string, ...string) (string, bool)
}
)
@@ -101,6 +102,7 @@ const (
sysID = "ID"
sysNamespaceID = "namespaceID"
sysModuleID = "moduleID"
sysRevision = "revision"
sysCreatedAt = "createdAt"
sysCreatedBy = "createdBy"
sysUpdatedAt = "updatedAt"
@@ -112,6 +114,7 @@ const (
colSysID = "id"
colSysNamespaceID = "rel_namespace"
colSysModuleID = "module_id"
colSysRevision = "revision"
colSysCreatedAt = "created_at"
colSysCreatedBy = "created_by"
colSysUpdatedAt = "updated_at"
@@ -125,6 +128,7 @@ var (
systemFields = slice.ToStringBoolMap([]string{
"recordID",
"ownedBy",
"revision",
"createdBy",
"createdAt",
"updatedBy",
@@ -1113,6 +1117,7 @@ func modulesForNamespace(ns *types.Namespace, mm types.ModuleSet) (out types.Mod
out = append(out, m)
}
}
return
}
@@ -1172,7 +1177,6 @@ func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace
var (
cm dal.ConnectionMeta
attrAux dal.AttributeSet
ok bool
)
for connectionID, modules := range modulesByConnection(modules...) {
@@ -1204,21 +1208,13 @@ func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace
Capabilities: mod.Config.DAL.Capabilities,
}
// - make the model ident
ident := cm.DefaultModelIdent
model.Ident = cm.DefaultModelIdent
if mod.Config.DAL.Partitioned {
tpl := mod.Config.DAL.PartitionFormat
if tpl == "" {
tpl = cm.DefaultPartitionFormat
}
ident, ok = ff.Format(ctx, tpl, formatterModuleParams(mod)...)
if !ok {
err = fmt.Errorf("invalid model ident generated: %s", ident)
model.Ident, err = makeModelIdent(ctx, ff, cm, mod, mod.Config.DAL.PartitionFormat)
if err != nil {
return
}
}
model.Ident = ident
// Convert user-defined fields to attributes
attrAux, err = moduleFieldsToAttributes(ctx, cm, ns, mod)
@@ -1235,12 +1231,53 @@ func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace
model.Attributes = append(model.Attributes, attrAux...)
out = append(out, model)
if mod.Config.RecordRevisions.Enabled {
rModel := revisions.Model()
// reuse the connection from the module
rModel.ConnectionID = connectionID
rModel.Resource = model.Resource
// a little trick to ensure revision model keeps t
// the same ID and to avoid collisions with the model
rModel.ResourceID = mod.ID + 1
if mod.Config.RecordRevisions.Ident == "" {
mod.Config.RecordRevisions.Ident = "compose_record_revisions"
}
rModel.Ident, err = makeModelIdent(ctx, ff, cm, mod, mod.Config.RecordRevisions.Ident)
if err != nil {
return
}
out = append(out, rModel)
}
}
}
return
}
func makeModelIdent(ctx context.Context, ff identFormatter, cm dal.ConnectionMeta, mod *types.Module, ident string) (_ string, err error) {
var (
ok bool
)
if ident == "" {
ident = cm.DefaultPartitionFormat
}
ident, ok = ff.Format(ctx, ident, formatterModuleParams(mod)...)
if !ok {
err = fmt.Errorf("invalid model ident generated: %s", ident)
return
}
return ident, nil
}
// moduleFieldsToAttributes converts all user-defined module fields to attributes
func moduleFieldsToAttributes(ctx context.Context, cm dal.ConnectionMeta, ns *types.Namespace, mod *types.Module) (out dal.AttributeSet, err error) {
out = make(dal.AttributeSet, 0, len(mod.Fields))
@@ -1295,6 +1332,10 @@ func partitionedModuleSystemFieldsToAttributes(cm dal.ConnectionMeta, mod *types
out = append(out, dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, modelFieldCodec(cm, mod, mf(sysNamespaceID, sysEnc.NamespaceID))))
}
if sysEnc.Revision != nil {
out = append(out, dal.FullAttribute(sysRevision, &dal.TypeID{}, modelFieldCodec(cm, mod, mf(sysRevision, sysEnc.Revision))))
}
if sysEnc.OwnedBy != nil {
out = append(out, dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, modelFieldCodec(cm, mod, mf(sysOwnedBy, sysEnc.OwnedBy))))
}
@@ -1335,6 +1376,8 @@ func defaultModuleSystemFieldsToAttributes() dal.AttributeSet {
dal.FullAttribute(sysModuleID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysModuleID}),
dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysNamespaceID}),
dal.FullAttribute(sysRevision, &dal.TypeNumber{}, &dal.CodecAlias{Ident: colSysRevision}),
dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysOwnedBy}),
dal.FullAttribute(sysCreatedAt, &dal.TypeTimestamp{}, &dal.CodecAlias{Ident: colSysCreatedAt}),
compose/service/record.go
+79 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/revisions"
"regexp"
"sort"
"strconv"
@@ -47,6 +48,8 @@ type (
namespace namespaceFinder
module moduleFinder
revisions *recordRevisions
formatter recordValuesFormatter
sanitizer recordValuesSanitizer
validator recordValuesValidator
@@ -86,6 +89,7 @@ type (
CanReadRecord(context.Context, *types.Record) bool
CanUpdateRecord(context.Context, *types.Record) bool
CanDeleteRecord(context.Context, *types.Record) bool
CanSearchRevisionsOnRecord(context.Context, *types.Record) bool
recordManageOwnerAccessController
recordValueAccessController
@@ -105,6 +109,7 @@ type (
Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error)
Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error)
FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error)
SearchRevisions(ctx context.Context, namespaceID, moduleID, recordID uint64) (dal.Iterator, error)
RecordExport(context.Context, types.RecordFilter) error
RecordImport(context.Context, error) error
@@ -177,6 +182,8 @@ func Record() *record {
namespace: DefaultNamespace,
module: DefaultModule,
revisions: &recordRevisions{revisions.Service(dal.Service())},
formatter: values.Formatter(),
sanitizer: values.Sanitizer(),
}
@@ -435,6 +442,43 @@ func (svc record) findSensitive(ctx context.Context, userID uint64, namespace *t
return
}
// SearchRevisions returns iterator for revisions of a record
//
func (svc record) SearchRevisions(ctx context.Context, namespaceID, moduleID, recordID uint64) (dal.Iterator, error) {
var (
aProps = &recordActionProps{record: &types.Record{NamespaceID: namespaceID, ModuleID: moduleID, ID: recordID}}
rec *types.Record
iter dal.Iterator
mod *types.Module
ns *types.Namespace
)
err := func() (err error) {
ns, mod, rec, err = loadRecordCombo(ctx, svc.store, svc.dal, namespaceID, moduleID, recordID)
if err != nil {
return
}
aProps.setModule(mod)
aProps.setNamespace(ns)
aProps.setRecord(rec)
if !mod.Config.RecordRevisions.Enabled {
return RecordErrRevisionsDisabledOnModule()
}
if !svc.ac.CanSearchRevisionsOnRecord(ctx, rec) {
return RecordErrNotAllowedToSearchRevisions()
}
iter, err = svc.revisions.search(ctx, rec)
return
}()
return iter, svc.recordAction(ctx, aProps, RecordActionSearchRevisions, err)
}
func (svc record) RecordImport(ctx context.Context, err error) error {
return svc.recordAction(ctx, &recordActionProps{}, RecordActionImport, err)
}
@@ -617,6 +661,13 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec
return
}
// store revision
if m.Config.RecordRevisions.Enabled {
if err = svc.revisions.created(ctx, new); err != nil {
return
}
}
if err = label.Create(ctx, svc.store, new); err != nil {
return
}
@@ -909,7 +960,18 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec
}
}
return dalutils.ComposeRecordUpdate(ctx, svc.dal, m, upd)
if err = dalutils.ComposeRecordUpdate(ctx, svc.dal, m, upd); err != nil {
return err
}
if m.Config.RecordRevisions.Enabled {
// Prepare record revision for update
if err = svc.revisions.updated(ctx, upd, old); err != nil {
return err
}
}
return nil
})
if err != nil {
@@ -982,6 +1044,7 @@ func (svc record) procCreate(ctx context.Context, invokerID uint64, m *types.Mod
// Reset values to new record
// to make sure nobody slips in something we do not want
new.ID = nextID()
new.Revision = 1
new.CreatedBy = invokerID
new.CreatedAt = *nowUTC()
new.UpdatedAt = nil
@@ -1023,6 +1086,8 @@ func (svc record) Update(ctx context.Context, upd *types.Record) (rec *types.Rec
//
// Both these points introduce external data that need to be checked fully in the same manner
func (svc record) procUpdate(ctx context.Context, invokerID uint64, m *types.Module, upd *types.Record, old *types.Record) (rve *types.RecordValueErrorSet) {
upd.Revision = old.Revision + 1
// Mark all values as updated (new)
upd.Values.SetUpdatedFlag(true)
@@ -1096,12 +1161,25 @@ func (svc record) delete(ctx context.Context, namespaceID, moduleID, recordID ui
// ensure module ref is set before running through records workflows and scripts
del.SetModule(m)
// deleted, revision need to be set when RecordBeforeDelete is triggered
del.DeletedAt = nowUTC()
del.DeletedBy = invokerID
del.Revision = del.Revision + 1
{
// Calling before-record-delete scripts
if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeDelete(nil, del, m, ns, nil, nil)); err != nil {
return nil, err
}
}
if m.Config.RecordRevisions.Enabled {
// Prepare record revision for update
if err = svc.revisions.softDeleted(ctx, del); err != nil {
return
}
}
if err = dalutils.ComposeRecordSoftDelete(ctx, svc.dal, m, del); err != nil {
return nil, err
}
compose/service/record_actions.gen.go
Generated
+92
View File
@@ -612,6 +612,26 @@ func RecordActionImport(props ...*recordActionProps) *recordAction {
return a
}
// RecordActionSearchRevisions returns "compose:record.searchRevisions" action
//
// This function is auto-generated.
//
func RecordActionSearchRevisions(props ...*recordActionProps) *recordAction {
a := &recordAction{
timestamp: time.Now(),
resource: "compose:record",
action: "searchRevisions",
log: "record revisions searched",
severity: actionlog.Notice,
}
if len(props) > 0 {
a.props = props[0]
}
return a
}
// RecordActionExport returns "compose:record.export" action
//
// This function is auto-generated.
@@ -1102,6 +1122,78 @@ func RecordErrNotAllowedToSearch(mm ...*recordActionProps) *errors.Error {
return e
}
// RecordErrNotAllowedToSearchRevisions returns "compose:record.notAllowedToSearchRevisions" as *errors.Error
//
//
// This function is auto-generated.
//
func RecordErrNotAllowedToSearchRevisions(mm ...*recordActionProps) *errors.Error {
var p = &recordActionProps{}
if len(mm) > 0 {
p = mm[0]
}
var e = errors.New(
errors.KindInternal,
p.Format("not allowed to search or list record revisions", nil),
errors.Meta("type", "notAllowedToSearchRevisions"),
errors.Meta("resource", "compose:record"),
// action log entry; no formatting, it will be applied inside recordAction fn.
errors.Meta(recordLogMetaKey{}, "failed to search or list record revisions; insufficient permissions"),
errors.Meta(recordPropsMetaKey{}, p),
// translation namespace & key
errors.Meta(locale.ErrorMetaNamespace{}, "compose"),
errors.Meta(locale.ErrorMetaKey{}, "record.errors.notAllowedToSearchRevisions"),
errors.StackSkip(1),
)
if len(mm) > 0 {
}
return e
}
// RecordErrRevisionsDisabledOnModule returns "compose:record.revisionsDisabledOnModule" as *errors.Error
//
//
// This function is auto-generated.
//
func RecordErrRevisionsDisabledOnModule(mm ...*recordActionProps) *errors.Error {
var p = &recordActionProps{}
if len(mm) > 0 {
p = mm[0]
}
var e = errors.New(
errors.KindInternal,
p.Format("revisions are disabled on module", nil),
errors.Meta("type", "revisionsDisabledOnModule"),
errors.Meta("resource", "compose:record"),
// action log entry; no formatting, it will be applied inside recordAction fn.
errors.Meta(recordLogMetaKey{}, "failed to search or list record revisions; disabled on module"),
errors.Meta(recordPropsMetaKey{}, p),
// translation namespace & key
errors.Meta(locale.ErrorMetaNamespace{}, "compose"),
errors.Meta(locale.ErrorMetaKey{}, "record.errors.revisionsDisabledOnModule"),
errors.StackSkip(1),
)
if len(mm) > 0 {
}
return e
}
// RecordErrNotAllowedToReadNamespace returns "compose:record.notAllowedToReadNamespace" as *errors.Error
//
//
compose/service/record_actions.yaml
+11
View File
@@ -72,6 +72,9 @@ actions:
- action: import
log: "records imported"
- action: searchRevisions
log: "record revisions searched"
- action: export
log: "records exported"
@@ -130,6 +133,14 @@ errors:
message: "not allowed to search or list records"
log: "failed to search or list records; insufficient permissions"
- error: notAllowedToSearchRevisions
message: "not allowed to search or list record revisions"
log: "failed to search or list record revisions; insufficient permissions"
- error: revisionsDisabledOnModule
message: "revisions are disabled on module"
log: "failed to search or list record revisions; disabled on module"
- error: notAllowedToReadNamespace
message: "not allowed to read this namespace"
log: "failed to read namespace {{namespace}}; insufficient permissions"
compose/service/record_revisions.go
+144
View File
@@ -0,0 +1,144 @@
package service
import (
"context"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/dal"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/revisions"
)
type (
// wrapper around revision.service
//
// handles all module-to-model conversion
recordRevisions struct {
r interface {
Search(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error)
Create(ctx context.Context, mf dal.ModelRef, revision *revisions.Revision) error
}
}
)
// constructs DAL model-ref from module
//
// Since we're using revisions model resource type needs to be altered.
func (svc *recordRevisions) modelRef(mod *types.Module) (mf dal.ModelRef) {
mf = mod.ModelRef()
// removing resource ID from model filter
// this will force DAL service to use FindModelByResourceIdent
// to find model with same Resource value and RevisionResourceType
// for ResourceType
mf.ResourceID = 0
mf.ResourceType = revisions.RevisionResourceType
return
}
func (svc *recordRevisions) search(ctx context.Context, rec *types.Record) (_ dal.Iterator, err error) {
return svc.r.Search(
ctx,
svc.modelRef(rec.GetModule()),
filter.Generic(filter.WithConstraint("rel_resource", rec.ID)),
)
}
func (svc *recordRevisions) created(ctx context.Context, new *types.Record) (err error) {
var (
skipList = svc.skippedField(new.GetModule())
invokerID = auth.GetIdentityFromContext(ctx).Identity()
rev *revisions.Revision
)
rev = revisions.Make(revisions.Created, new.Revision, new.ID, invokerID)
err = rev.CollectChanges(new, nil, skipList...)
if err != nil {
return
}
return svc.r.Create(ctx, svc.modelRef(new.GetModule()), rev)
}
func (svc *recordRevisions) updated(ctx context.Context, upd, old *types.Record) (err error) {
var (
skipList = svc.skippedField(old.GetModule())
invokerID = auth.GetIdentityFromContext(ctx).Identity()
rev *revisions.Revision
)
rev = revisions.Make(revisions.Updated, upd.Revision, upd.ID, invokerID)
err = rev.CollectChanges(upd, old, skipList...)
if err != nil {
return
}
return svc.r.Create(ctx, svc.modelRef(upd.GetModule()), rev)
}
func (svc *recordRevisions) softDeleted(ctx context.Context, del *types.Record) (err error) {
var (
invokerID = auth.GetIdentityFromContext(ctx).Identity()
rev *revisions.Revision
)
rev = revisions.Make(revisions.SoftDeleted, del.Revision, del.ID, invokerID)
if err != nil {
return
}
return svc.r.Create(ctx, svc.modelRef(del.GetModule()), rev)
}
func (svc *recordRevisions) skippedField(mod *types.Module) []string {
list := []string{
"ID",
"namespaceID",
"moduleID",
"revision",
"createdBy",
"createdAt",
"updatedBy",
"updatedAt",
"deletedBy",
"deletedAt",
}
for _, f := range mod.Fields {
if f.Config.RecordRevisions.Skip {
list = append(list, f.Name)
}
}
return list
}
// @todo uncomment when supported
//func (svc *recordRevisions) restored(ctx context.Context, del *types.Record) (err error) {
// var (
// invokerID = auth.GetIdentityFromContext(ctx).Identity()
// rev *revisions.Revision
// )
//
// rev, err = revisions.Make(revisions.Restored, del.Revision, del.ID, invokerID, nil, nil)
// if err != nil {
// return
// }
//
// return svc.r.Create(ctx, svc.modelRef(del.GetModule()), rev)
//}
// @todo uncomment when supported
//func (svc *recordRevisions) hardDeleted(ctx context.Context, del *types.Record) (err error) {
// var (
// invokerID = auth.GetIdentityFromContext(ctx).Identity()
// rev *revisions.Revision
// )
//
// rev, err = revisions.Make(revisions.HardDeleted, del.Revision, del.ID, invokerID, nil, nil)
// if err != nil {
// return
// }
//
// return svc.r.Create(ctx, svc.modelRef(del.GetModule()), rev)
//}
compose/service/record_revisions_test.go
+63
View File
@@ -0,0 +1,63 @@
package service
import (
"context"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/dal"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/revisions"
"github.com/stretchr/testify/require"
"testing"
)
type (
mockRecordRevisionsDAL struct {
search func(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error)
create func(ctx context.Context, mf dal.ModelRef, revision *revisions.Revision) error
}
)
func (svc *mockRecordRevisionsDAL) Search(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error) {
return svc.search(ctx, mf, f)
}
func (svc *mockRecordRevisionsDAL) Create(ctx context.Context, mf dal.ModelRef, rev *revisions.Revision) error {
return svc.create(ctx, mf, rev)
}
func TestRecordRevisions(t *testing.T) {
var (
req = require.New(t)
ctx = context.Background()
skip = types.ModuleFieldConfig{RecordRevisions: types.ModuleFieldConfigRecordRevisions{Skip: true}}
mod = &types.Module{
ID: 1,
Name: "test",
Fields: []*types.ModuleField{
{Name: "rev1", Kind: "string"},
{Name: "rev2", Kind: "string"},
{Name: "rev3", Kind: "string", Config: skip},
},
}
rec = &types.Record{ID: 2, ModuleID: 1}
dalMock = &mockRecordRevisionsDAL{}
svc = &recordRevisions{r: dalMock}
)
rec.SetModule(mod)
req.NoError(rec.SetValue("rev1", 0, "val1"))
req.NoError(rec.SetValue("rev2", 0, "val2"))
req.NoError(rec.SetValue("rev3", 0, "val3"))
var changes int
dalMock.create = func(ctx context.Context, mf dal.ModelRef, r *revisions.Revision) error {
changes = len(r.Changes)
return nil
}
req.NoError(svc.created(ctx, rec))
req.Equal(3, changes, "expecting ownedBy, rev1 and rev2")
}
compose/service/record_test.go
+2 -2
View File
@@ -507,8 +507,8 @@ func TestRecord_refAccessControl(t *testing.T) {
req = require.New(t)
// uncomment to enable sql conn debugging
//ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger())
ctx = context.Background()
ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger())
//ctx = context.Background()
s, err = sqlite.ConnectInMemoryWithDebug(ctx)
)
compose/types/module.go
+12
View File
@@ -51,6 +51,8 @@ type (
ModuleID *EncodingStrategy `json:"moduleID"`
NamespaceID *EncodingStrategy `json:"namespaceID"`
Revision *EncodingStrategy `json:"revision"`
OwnedBy *EncodingStrategy `json:"ownedBy"`
CreatedAt *EncodingStrategy `json:"createdAt"`
@@ -72,6 +74,8 @@ type (
// @todo we need to transfer this from meta!!
Discovery discovery.ModuleMeta `json:"discovery"`
RecordRevisions ModuleConfigRecordRevisions `json:"recordRevisions"`
}
ModuleConfigDAL struct {
@@ -88,6 +92,14 @@ type (
SystemFieldEncoding SystemFieldEncoding `json:"systemFieldEncoding"`
}
ModuleConfigRecordRevisions struct {
// enable or disable revisions
Enabled bool `json:"enabled"`
// where are record revisions stored
Ident string `json:"ident"`
}
ModuleConfigDataPrivacy struct {
// Define the highest sensitivity level which
// can be configured on the module fields
compose/types/module_field.go
+7
View File
@@ -53,6 +53,8 @@ type (
ModuleFieldConfig struct {
DAL ModuleFieldConfigDAL `json:"dal"`
Privacy ModuleFieldConfigDataPrivacy `json:"privacy"`
RecordRevisions ModuleFieldConfigRecordRevisions `json:"recordRevisions"`
}
ModuleFieldConfigDAL struct {
@@ -67,6 +69,11 @@ type (
UsageDisclosure string `json:"usageDisclosure"`
}
ModuleFieldConfigRecordRevisions struct {
// when true, skip record revisions for this field
Skip bool `json:"enabled"`
}
EncodingStrategy struct {
*EncodingStrategyAlias `json:"alias,omitempty"`
*EncodingStrategyJSON `json:"json,omitempty"`
compose/types/record.go
+34 -20
View File
@@ -37,6 +37,8 @@ type (
ID uint64 `json:"recordID,string"`
ModuleID uint64 `json:"moduleID,string"`
Revision uint `json:"revision,omitempty"`
module *Module
Values RecordValueSet `json:"values,omitempty"`
@@ -49,9 +51,9 @@ type (
CreatedAt time.Time `json:"createdAt,omitempty"`
CreatedBy uint64 `json:"createdBy,string" `
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
UpdatedBy uint64 `json:"updatedBy,string,omitempty" `
UpdatedBy uint64 `json:"updatedBy,string,omitempty"`
DeletedAt *time.Time `json:"deletedAt,omitempty"`
DeletedBy uint64 `json:"deletedBy,string,omitempty" `
DeletedBy uint64 `json:"deletedBy,string,omitempty"`
}
RecordFilter struct {
@@ -193,6 +195,8 @@ func (r *Record) GetValue(name string, pos uint) (any, error) {
return r.ModuleID, nil
case "namespaceID":
return r.NamespaceID, nil
case "revision":
return r.Revision, nil
case "createdAt":
return r.CreatedAt, nil
case "createdBy":
@@ -217,33 +221,40 @@ func (r *Record) GetValue(name string, pos uint) (any, error) {
}
// CountValues returns how many values per field are there
func (r *Record) CountValues() map[string]uint {
func (r *Record) CountValues() (pos map[string]uint) {
var (
pos = map[string]uint{
"ID": 1,
"moduleID": 1,
"namespaceID": 1,
"createdAt": 1,
"createdBy": 1,
"updatedAt": 1,
"updatedBy": 1,
"deletedAt": 1,
"deletedBy": 1,
"ownedBy": 1,
}
mod = r.GetModule()
)
if mod := r.GetModule(); mod != nil {
for _, f := range mod.Fields {
pos[f.Name] = 0
}
pos = map[string]uint{
"ID": 1,
"moduleID": 1,
"namespaceID": 1,
"revision": 1,
"createdAt": 1,
"createdBy": 1,
"updatedAt": 1,
"updatedBy": 1,
"deletedAt": 1,
"deletedBy": 1,
"ownedBy": 1,
}
if mod == nil {
// count record values
// only when module is known
return
}
for _, f := range mod.Fields {
pos[f.Name] = 0
}
for _, val := range r.Values {
pos[val.Name]++
}
return pos
return
}
func (r *Record) SetValue(name string, pos uint, value any) (err error) {
@@ -262,6 +273,8 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) {
return cast2.Uint64(value, &r.DeletedBy)
case "ownedBy":
return cast2.Uint64(value, &r.OwnedBy)
case "revision":
return cast2.Uint(value, &r.Revision)
case "createdAt":
return cast2.Time(value, &r.CreatedAt)
case "updatedAt":
@@ -323,6 +336,7 @@ func (r Record) Dict() map[string]interface{} {
"ID": r.ID,
"recordID": r.ID,
"moduleID": r.ModuleID,
"revision": r.Revision,
"labels": r.Labels,
"namespaceID": r.NamespaceID,
"ownedBy": r.OwnedBy,
pkg/revisions/revisions.go
+3 -3
View File
@@ -59,7 +59,7 @@ func Make(op Operation, revision uint, resourceID, userID uint64) (rev *Revision
}
// untested
func (r *Revision) CollectChanges(new, old dal.ValueGetter, omit ...string) (err error) {
func (r *Revision) CollectChanges(new, old dal.ValueGetter, skip ...string) (err error) {
var (
cc = make([]*Change, 0)
ch *Change
@@ -86,8 +86,8 @@ keys:
continue keys
}
for _, o := range omit {
if key == o {
for _, s := range skip {
if key == s {
continue keys
}
}
provision/000_base/compose_access_control.yaml
+1
View File
@@ -60,6 +60,7 @@ allow:
- read
- update
- delete
- revisions.search
corteza::compose:chart/*/*:
- read
store/adapters/rdbms/dal/iterator.go
-1
View File
@@ -161,7 +161,6 @@ func (i *iterator) fetch(ctx context.Context) (_ *sql.Rows, err error) {
}
}
sql, _, _ = query.Prepared(false).ToSQL()
if sql, args, err = query.ToSQL(); err != nil {
return nil, err
}
store/adapters/rdbms/upgrade.go
+1
View File
@@ -30,6 +30,7 @@ func (s *Store) Upgrade(ctx context.Context) (err error) {
fix202209_extendComposeModuleForPrivacyAndDAL,
fix202209_extendComposeModuleFieldsForPrivacyAndDAL,
fix202209_dropObsoleteComposeModuleFields,
fix202209_composeRecordRevisions,
}
for _, fix := range fixes {
store/adapters/rdbms/upgrade_fixes.go
+84
View File
@@ -32,3 +32,87 @@ func fix202209_dropObsoleteComposeModuleFields(ctx context.Context, s *Store) (e
"is_visible",
)
}
func fix202209_composeRecordRevisions(ctx context.Context, s *Store) (err error) {
s.log(ctx).Info("extending compose_record table with revision column")
return s.SchemaAPI.AddColumn(
ctx, s.DB,
&Table{Name: "compose_record"},
&Column{Type: ColumnType{Type: ColumnTypeInteger}, DefaultValue: "1", Name: "revision"},
)
//return // will probably not need this...
//var (
// log = s.log(ctx)
//)
//
//const (
// envKeySkip = "UPGRADE_COMPOSE_RECORD_CHANGES_PREFILL_SKIP"
//
// tblChanges = "compose_record_changes"
// tblRecords = "compose_record"
//)
//
//if _, set := os.LookupEnv(envKeySkip); set {
// return
//}
//
//var (
// tblRecordColumns = []any{
// // reusing record ID for ID of the (1st record)
// // entry in the changes table
// "id",
// "id",
// "rel_namespace", "module_id",
// "values",
// "owned_by",
// "created_at", "created_by",
// "updated_at", "updated_by",
// "deleted_at", "deleted_by",
// }
//
// tblChangesColumns = []any{
// "id",
// "rel_record",
// "rel_namespace", "rel_module",
// "values",
// "owned_by",
// "created_at", "created_by",
// "updated_at", "updated_by",
// "deleted_at", "deleted_by",
// }
//
// d = s.Dialect
// check = d.Select(exp.NewLiteralExpression("1")).From(tblChanges).Limit(1)
// count int
//
// copyAll = d.
// Insert(tblChanges).
// Cols(tblChangesColumns...).
// FromQuery(d.From(tblRecords).Select(tblRecordColumns...))
//)
//
//if err = s.QueryOne(ctx, check, &count); err != nil {
// // exit on error or when changes table is not empty
// if !errors.IsNotFound(err) {
// return fmt.Errorf("could not check if %q is empty: %w", tblChanges, err)
// }
//
//} else if count > 0 {
// return
//}
//
//log.Warn(fmt.Sprintf("Empty %s table detected. "+
// "Prefilling record changes table with current records, "+
// "this might take a while, depending amount of records you have and how fast your database is. "+
// "If you need to disable this, stop the server, set %s=1 "+
// "and restart the server.",
// tblChanges,
// envKeySkip,
//))
//
//spew.Dump(copyAll.ToSQL())
//
//// make a copy of records to changes table
//return s.Exec(ctx, copyAll)
}
store/adapters/rdbms/upgrade_tables.go
+17
View File
@@ -59,6 +59,7 @@ func Tables() []*Table {
tableComposeNamespace(),
tableComposePage(),
tableComposeRecord(),
tableComposeRecordRevisions(),
tableFederationModuleShared(),
tableFederationModuleExposed(),
tableFederationModuleMapping(),
@@ -519,6 +520,7 @@ func tableComposeRecord() *Table {
ID,
ColumnDef("rel_namespace", ColumnTypeIdentifier),
ColumnDef("module_id", ColumnTypeIdentifier),
ColumnDef("revision", ColumnTypeInteger),
ColumnDef("values", ColumnTypeJson),
ColumnDef("owned_by", ColumnTypeIdentifier),
CUDTimestamps,
@@ -530,6 +532,21 @@ func tableComposeRecord() *Table {
)
}
func tableComposeRecordRevisions() *Table {
return TableDef("compose_record_revisions",
ID,
ColumnDef("ts", ColumnTypeTimestamp),
ColumnDef("rel_resource", ColumnTypeIdentifier),
ColumnDef("revision", ColumnTypeInteger),
ColumnDef("operation", ColumnTypeText),
ColumnDef("rel_user", ColumnTypeIdentifier),
ColumnDef("delta", ColumnTypeJson),
ColumnDef("comment", ColumnTypeText),
AddIndex("record", IColumn("rel_resource")),
)
}
func tableFederationModuleShared() *Table {
return TableDef("federation_module_shared",
ID,