From 508dddd3048a0eed632ee26aa5dfc15f0dac9eb1 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Wed, 27 Jul 2022 20:16:44 +0200 Subject: [PATCH 1/3] Implement generic package for revisions --- pkg/revisions/model.go | 40 +++++++ pkg/revisions/operations.go | 14 +++ pkg/revisions/revisions.go | 225 ++++++++++++++++++++++++++++++++++++ pkg/revisions/service.go | 37 ++++++ 4 files changed, 316 insertions(+) create mode 100644 pkg/revisions/model.go create mode 100644 pkg/revisions/operations.go create mode 100644 pkg/revisions/revisions.go create mode 100644 pkg/revisions/service.go diff --git a/pkg/revisions/model.go b/pkg/revisions/model.go new file mode 100644 index 000000000..39faad5df --- /dev/null +++ b/pkg/revisions/model.go @@ -0,0 +1,40 @@ +package revisions + +import ( + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" + "github.com/cortezaproject/corteza-server/pkg/id" +) + +const ( + RevisionResourceType = "corteza::system:revision" +) + +// Model returns generic dal.Model for storing revisions +// +// Returns only basic +func Model() *dal.Model { + // make revision model + return &dal.Model{ + ResourceID: id.Next(), + + Ident: "revisions", + + ResourceType: RevisionResourceType, + Attributes: dal.AttributeSet{ + &dal.Attribute{Ident: "id", PrimaryKey: true, Store: &dal.CodecPlain{}, Type: &dal.TypeID{}}, + &dal.Attribute{Ident: "ts", Store: &dal.CodecPlain{}, Type: &dal.TypeTimestamp{}}, + &dal.Attribute{Ident: "revision", Store: &dal.CodecPlain{}, Type: &dal.TypeNumber{}}, + &dal.Attribute{Ident: "operation", Store: &dal.CodecPlain{}, Type: &dal.TypeNumber{}}, + &dal.Attribute{Ident: "rel_resource", Store: &dal.CodecPlain{}, Type: &dal.TypeID{}}, + &dal.Attribute{Ident: "rel_user", Store: &dal.CodecPlain{}, Type: &dal.TypeID{}}, + &dal.Attribute{Ident: "delta", Store: &dal.CodecPlain{}, Type: &dal.TypeJSON{}}, + &dal.Attribute{Ident: "comment", Store: &dal.CodecPlain{}, Type: &dal.TypeText{}}, + }, + + Capabilities: capabilities.Set{ + capabilities.Create, + capabilities.Search, + }, + } +} diff --git a/pkg/revisions/operations.go b/pkg/revisions/operations.go new file mode 100644 index 000000000..bb83d1f3b --- /dev/null +++ b/pkg/revisions/operations.go @@ -0,0 +1,14 @@ +package revisions + +type ( + Operation = string +) + +const ( + unknown Operation = "" + Created = "created" + Updated = "updated" + SoftDeleted = "soft-deleted" + Restored = "restored" + HardDeleted = "hard-deleted" +) diff --git a/pkg/revisions/revisions.go b/pkg/revisions/revisions.go new file mode 100644 index 000000000..58d17daa8 --- /dev/null +++ b/pkg/revisions/revisions.go @@ -0,0 +1,225 @@ +package revisions + +import ( + "encoding/json" + "fmt" + "github.com/cortezaproject/corteza-server/pkg/cast2" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/id" + "time" +) + +type ( + // generic revision struct + Revision struct { + ID uint64 `json:"changeID,string"` + Timestamp time.Time `json:"timestamp"` + + ResourceID uint64 `json:"resourceID,string"` + + Revision uint `json:"revision"` + Operation Operation `json:"event"` + + UserID uint64 `json:"userID,string"` + + Changes []*Change `json:"changes"` + + Comment string `json:"comment"` + } + + Change struct { + // changed field + Key string `json:"key"` + + Old []any `json:"old,omitempty"` + New []any `json:"new,omitempty"` + } + + Filter struct { + ResourceID uint64 `json:"resourceID,string"` + } +) + +var ( + now = func() time.Time { + return time.Now().Round(time.Second) + } +) + +func Make(op Operation, revision uint, resourceID, userID uint64) (rev *Revision) { + return &Revision{ + ID: id.Next(), + Timestamp: now(), + ResourceID: resourceID, + Revision: revision, + UserID: userID, + Operation: op, + } +} + +// untested +func (r *Revision) CollectChanges(new, old dal.ValueGetter, omit ...string) (err error) { + var ( + cc = make([]*Change, 0) + ch *Change + + // old values count + ovc map[string]uint + + val any + ) + + if new == nil { + return + } + + if old != nil { + // old values count + ovc = old.CountValues() + + } + +keys: + for key, count := range new.CountValues() { + if count == 0 && old == nil { + continue keys + } + + for _, o := range omit { + if key == o { + continue keys + } + } + + ch = &Change{ + Key: key, + New: make([]any, 0, count), + Old: make([]any, 0, count), + } + + for pos := uint(0); pos < count; pos++ { + val, err = new.GetValue(key, pos) + if err != nil { + return + } + + ch.New = append(ch.New, val) + } + + if old == nil { + cc = append(cc, ch) + continue + } + + for pos := uint(0); pos < ovc[key]; pos++ { + val, err = old.GetValue(key, pos) + if err != nil { + return + } + + ch.Old = append(ch.Old, val) + } + + if len(ch.New) != len(ch.Old) { + // different sizes, means that something has changed + cc = append(cc, ch) + continue + } + + for i := range ch.New { + // go over all and append on first found difference + if ch.New[i] != ch.Old[i] { + cc = append(cc, ch) + break + } + } + } + + r.Changes = cc + return +} + +// CountValues satisfies dal.ValueGetter interface +func (r *Revision) CountValues() map[string]uint { + // signaling DAL that each attribute has exactly one value! + return nil +} + +// GetValue satisfies dal.ValueGetter interface +func (r *Revision) GetValue(ident string, _ uint) (any, error) { + switch ident { + case "id": + return r.ID, nil + + case "ts": + return r.Timestamp, nil + + case "rel_resource": + return r.ResourceID, nil + + case "revision": + return r.Revision, nil + + case "operation": + return r.Operation, nil + + case "rel_user": + return r.UserID, nil + + case "delta": + return r.Changes, nil + + case "comment": + return r.Comment, nil + + } + + return nil, nil +} + +func (r *Revision) SetValue(name string, _ uint, value any) error { + switch name { + case "id": + return cast2.Uint64(value, &r.ID) + + case "ts": + return cast2.Time(value, &r.Timestamp) + + case "rel_resource": + return cast2.Uint64(value, &r.ResourceID) + + case "revision": + return cast2.Uint(value, &r.Revision) + + case "operation": + return cast2.String(value, &r.Operation) + + case "rel_user": + return cast2.Uint64(value, &r.UserID) + + case "delta": + if bb, is := value.([]byte); is { + return json.Unmarshal(bb, &r.Changes) + } + + return fmt.Errorf("unexpected type for delta: %T", value) + + case "comment": + return cast2.String(value, &r.Comment) + } + + return nil +} + +func (f Filter) Constraints() map[string][]any { + return map[string][]any{ + "rel_resource": {f.ResourceID}, + } +} + +func (f Filter) Expression() string { return "" } +func (f Filter) OrderBy() filter.SortExprSet { return nil } +func (f Filter) Limit() uint { return 0 } +func (f Filter) Cursor() *filter.PagingCursor { return nil } +func (f Filter) StateConstraints() map[string]filter.State { return nil } diff --git a/pkg/revisions/service.go b/pkg/revisions/service.go new file mode 100644 index 000000000..bf84316f7 --- /dev/null +++ b/pkg/revisions/service.go @@ -0,0 +1,37 @@ +package revisions + +import ( + "context" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/dal/capabilities" + "github.com/cortezaproject/corteza-server/pkg/filter" +) + +type ( + Servicer interface { + Search(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error) + Create(ctx context.Context, mf dal.ModelRef, revision *Revision) error + } + + creatorSearcher interface { + Search(ctx context.Context, m dal.ModelRef, capabilities capabilities.Set, f filter.Filter) (dal.Iterator, error) + Create(ctx context.Context, m dal.ModelRef, capabilities capabilities.Set, vv ...dal.ValueGetter) error + } + + service struct { + dal creatorSearcher + } +) + +func Service(dal creatorSearcher) *service { + return &service{dal: dal} +} + +func (svc *service) Search(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error) { + return svc.dal.Search(ctx, mf, capabilities.Set{capabilities.Search}, f) +} + +func (svc *service) Create(ctx context.Context, mf dal.ModelRef, revision *Revision) error { + return svc.dal.Create(ctx, mf, capabilities.Set{capabilities.Create}, revision) + +} From 39bf9b75ea1556b57a2a85f24035809e24bce64f Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Thu, 28 Jul 2022 20:03:24 +0200 Subject: [PATCH 2/3] Deprecate ACTIONLOG_COMPOSE_RECORD_ENABLED --- .env.example | 13 ------------- app/options/actionLog.cue | 14 -------------- compose/service/service.go | 13 ------------- pkg/options/options.gen.go | 4 +--- 4 files changed, 1 insertion(+), 43 deletions(-) diff --git a/.env.example b/.env.example index a95925960..3714c23f6 100644 --- a/.env.example +++ b/.env.example @@ -346,19 +346,6 @@ # Default: # ACTIONLOG_WORKFLOW_FUNCTIONS_ENABLED= -############################################################################### -# Enables actionlog for compose record create, update, and delete. which is disabled by default. -# -# [IMPORTANT] -# ==== -# This is temp fix for now, it will be removed completely in future release. -# Once new env var will be introduced for actionlog policy, which will enable more control over action log policies. -# ==== -# -# Type: bool -# Default: -# ACTIONLOG_COMPOSE_RECORD_ENABLED= - ############################################################################### ############################################################################### # API Gateway diff --git a/app/options/actionLog.cue b/app/options/actionLog.cue index e41880df3..8784387e7 100644 --- a/app/options/actionLog.cue +++ b/app/options/actionLog.cue @@ -18,20 +18,6 @@ actionLog: schema.#optionsGroup & { workflow_functions_enabled: { type: "bool" } - compose_record_enabled: { - type: "bool" - defaultGoExpr: "false" - description: """ - Enables actionlog for compose record create, update, and delete. which is disabled by default. - - [IMPORTANT] - ==== - This is temp fix for now, it will be removed completely in future release. - Once new env var will be introduced for actionlog policy, which will enable more control over action log policies. - ==== - - """ - } } title: "Actionlog" } diff --git a/compose/service/service.go b/compose/service/service.go index 169f0d08e..4fc2a7c63 100644 --- a/compose/service/service.go +++ b/compose/service/service.go @@ -112,19 +112,6 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, c Config) tee = logger.MakeDebugLogger() } - // Record log policy update - if !c.ActionLog.ComposeRecordEnabled { - policy = actionlog.NewPolicyAll( - policy, - - actionlog.NewPolicyAll( - actionlog.NewPolicyMatchResource("compose:record"), - actionlog.NewPolicyMatchAction("create", "update", "delete"), - actionlog.NewPolicyMatchSeverity(actionlog.Emergency), - ), - ) - } - DefaultActionlog = actionlog.NewService(DefaultStore, log, tee, policy) } diff --git a/pkg/options/options.gen.go b/pkg/options/options.gen.go index 947d16ad0..81992e2b7 100644 --- a/pkg/options/options.gen.go +++ b/pkg/options/options.gen.go @@ -79,7 +79,6 @@ type ( Enabled bool `env:"ACTIONLOG_ENABLED"` Debug bool `env:"ACTIONLOG_DEBUG"` WorkflowFunctionsEnabled bool `env:"ACTIONLOG_WORKFLOW_FUNCTIONS_ENABLED"` - ComposeRecordEnabled bool `env:"ACTIONLOG_COMPOSE_RECORD_ENABLED"` } ApigwOpt struct { @@ -456,8 +455,7 @@ func SMTP() (o *SMTPOpt) { // This function is auto-generated func ActionLog() (o *ActionLogOpt) { o = &ActionLogOpt{ - Enabled: true, - ComposeRecordEnabled: false, + Enabled: true, } // Custom defaults From c9442260c54a001337a067bcdc37b4e9ae927ad6 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Wed, 27 Jul 2022 20:17:02 +0200 Subject: [PATCH 3/3] Add support for revisions on compose records --- compose/record.cue | 1 + compose/rest.yaml | 7 + compose/rest/handlers/record.go | 20 +++ compose/rest/record.go | 36 ++++- compose/rest/request/record.go | 76 +++++++++ compose/service/access_control.gen.go | 21 ++- compose/service/module.go | 71 +++++++-- compose/service/record.go | 80 +++++++++- compose/service/record_actions.gen.go | 92 +++++++++++ compose/service/record_actions.yaml | 11 ++ compose/service/record_revisions.go | 144 ++++++++++++++++++ compose/service/record_revisions_test.go | 63 ++++++++ compose/service/record_test.go | 4 +- compose/types/module.go | 12 ++ compose/types/module_field.go | 7 + compose/types/record.go | 54 ++++--- pkg/revisions/revisions.go | 6 +- .../000_base/compose_access_control.yaml | 1 + store/adapters/rdbms/dal/iterator.go | 1 - store/adapters/rdbms/upgrade.go | 1 + store/adapters/rdbms/upgrade_fixes.go | 84 ++++++++++ store/adapters/rdbms/upgrade_tables.go | 17 +++ 22 files changed, 763 insertions(+), 46 deletions(-) create mode 100644 compose/service/record_revisions.go create mode 100644 compose/service/record_revisions_test.go diff --git a/compose/record.cue b/compose/record.cue index f10f66c1c..c20988922 100644 --- a/compose/record.cue +++ b/compose/record.cue @@ -43,6 +43,7 @@ record: schema.#Resource & { "update": {} "delete": {} "owner.manage": {} + "revisions.search": {} } } } diff --git a/compose/rest.yaml b/compose/rest.yaml index abbe1deb1..aef289979 100644 --- a/compose/rest.yaml +++ b/compose/rest.yaml @@ -922,6 +922,13 @@ endpoints: type: map[string]interface{} parser: parseMapStringInterface title: Arguments to pass to the script + - name: revisions + method: GET + title: List record revisions + path: "/{recordID}/revisions" + parameters: + path: + - { type: uint64, name: recordID, required: true, title: ID } - title: Data Privacy entrypoint: dataPrivacy diff --git a/compose/rest/handlers/record.go b/compose/rest/handlers/record.go index 1e4a9e5d3..a25e14b38 100644 --- a/compose/rest/handlers/record.go +++ b/compose/rest/handlers/record.go @@ -34,6 +34,7 @@ type ( Upload(context.Context, *request.RecordUpload) (interface{}, error) TriggerScript(context.Context, *request.RecordTriggerScript) (interface{}, error) TriggerScriptOnList(context.Context, *request.RecordTriggerScriptOnList) (interface{}, error) + Revisions(context.Context, *request.RecordRevisions) (interface{}, error) } // HTTP API interface @@ -53,6 +54,7 @@ type ( Upload func(http.ResponseWriter, *http.Request) TriggerScript func(http.ResponseWriter, *http.Request) TriggerScriptOnList func(http.ResponseWriter, *http.Request) + Revisions func(http.ResponseWriter, *http.Request) } ) @@ -296,6 +298,23 @@ func NewRecord(h RecordAPI) *Record { return } + api.Send(w, r, value) + }, + Revisions: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewRecordRevisions() + if err := params.Fill(r); err != nil { + api.Send(w, r, err) + return + } + + value, err := h. + Revisions(r.Context(), params) + if err != nil { + api.Send(w, r, err) + return + } + api.Send(w, r, value) }, } @@ -319,5 +338,6 @@ func (h Record) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http r.Post("/namespace/{namespaceID}/module/{moduleID}/record/attachment", h.Upload) r.Post("/namespace/{namespaceID}/module/{moduleID}/record/{recordID}/trigger", h.TriggerScript) r.Post("/namespace/{namespaceID}/module/{moduleID}/record/trigger", h.TriggerScriptOnList) + r.Get("/namespace/{namespaceID}/module/{moduleID}/record/{recordID}/revisions", h.Revisions) }) } diff --git a/compose/rest/record.go b/compose/rest/record.go index 302cf20b5..6c01af9b8 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/cortezaproject/corteza-server/pkg/revisions" "io" "math" "net/http" @@ -37,7 +38,9 @@ type ( CanUpdateRecord bool `json:"canUpdateRecord"` CanReadRecord bool `json:"canReadRecord"` CanDeleteRecord bool `json:"canDeleteRecord"` - CanGrant bool `json:"canGrant"` + CanSearchRevisions bool `json:"canSearchRevisions"` + + CanGrant bool `json:"canGrant"` } recordSetPayload struct { @@ -61,6 +64,7 @@ type ( CanReadRecord(context.Context, *types.Record) bool CanDeleteRecord(context.Context, *types.Record) bool CanManageOwnerOnRecord(context.Context, *types.Record) bool + CanSearchRevisionsOnRecord(context.Context, *types.Record) bool } ) @@ -567,6 +571,34 @@ func (ctrl *Record) TriggerScriptOnList(ctx context.Context, r *request.RecordTr return api.OK(), err } +func (ctrl *Record) Revisions(ctx context.Context, r *request.RecordRevisions) (interface{}, error) { + var ( + makeRev = func() dal.ValueSetter { return &revisions.Revision{} } + ) + + iter, err := ctrl.record.SearchRevisions(ctx, r.NamespaceID, r.ModuleID, r.RecordID) + if err != nil { + return nil, err + } + + return func(w http.ResponseWriter, req *http.Request) { + if _, err = w.Write([]byte(`{"set":[`)); err != nil { + return + } + + err = dal.IteratorEncodeJSON(ctx, w, iter, makeRev) + if err != nil { + return + } + + if _, err = w.Write([]byte(`]}`)); err != nil { + return + } + + return + }, err +} + func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err error, rr ...*types.Record) (*recordPayload, error) { if err != nil || rr == nil { return nil, err @@ -580,6 +612,7 @@ func (ctrl Record) makeBulkPayload(ctx context.Context, m *types.Module, err err CanUpdateRecord: ctrl.ac.CanUpdateRecord(ctx, rr[0]), CanReadRecord: ctrl.ac.CanReadRecord(ctx, rr[0]), CanDeleteRecord: ctrl.ac.CanDeleteRecord(ctx, rr[0]), + CanSearchRevisions: ctrl.ac.CanSearchRevisionsOnRecord(ctx, rr[0]), }, nil } @@ -597,6 +630,7 @@ func (ctrl Record) makePayload(ctx context.Context, m *types.Module, r *types.Re CanUpdateRecord: ctrl.ac.CanUpdateRecord(ctx, r), CanReadRecord: ctrl.ac.CanReadRecord(ctx, r), CanDeleteRecord: ctrl.ac.CanDeleteRecord(ctx, r), + CanSearchRevisions: ctrl.ac.CanSearchRevisionsOnRecord(ctx, r), }, nil } diff --git a/compose/rest/request/record.go b/compose/rest/request/record.go index c5bec5750..6837c6957 100644 --- a/compose/rest/request/record.go +++ b/compose/rest/request/record.go @@ -434,6 +434,23 @@ type ( // Arguments to pass to the script Args map[string]interface{} } + + RecordRevisions struct { + // NamespaceID PATH parameter + // + // Namespace ID + NamespaceID uint64 `json:",string"` + + // ModuleID PATH parameter + // + // Module ID + ModuleID uint64 `json:",string"` + + // RecordID PATH parameter + // + // ID + RecordID uint64 `json:",string"` + } ) // NewRecordReport request @@ -2085,3 +2102,62 @@ func (r *RecordTriggerScriptOnList) Fill(req *http.Request) (err error) { return err } + +// NewRecordRevisions request +func NewRecordRevisions() *RecordRevisions { + return &RecordRevisions{} +} + +// Auditable returns all auditable/loggable parameters +func (r RecordRevisions) Auditable() map[string]interface{} { + return map[string]interface{}{ + "namespaceID": r.NamespaceID, + "moduleID": r.ModuleID, + "recordID": r.RecordID, + } +} + +// Auditable returns all auditable/loggable parameters +func (r RecordRevisions) GetNamespaceID() uint64 { + return r.NamespaceID +} + +// Auditable returns all auditable/loggable parameters +func (r RecordRevisions) GetModuleID() uint64 { + return r.ModuleID +} + +// Auditable returns all auditable/loggable parameters +func (r RecordRevisions) GetRecordID() uint64 { + return r.RecordID +} + +// Fill processes request and fills internal variables +func (r *RecordRevisions) Fill(req *http.Request) (err error) { + + { + var val string + // path params + + val = chi.URLParam(req, "namespaceID") + r.NamespaceID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + + val = chi.URLParam(req, "moduleID") + r.ModuleID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + + val = chi.URLParam(req, "recordID") + r.RecordID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + + } + + return err +} diff --git a/compose/service/access_control.gen.go b/compose/service/access_control.gen.go index 31426f4a5..e127e5fb0 100644 --- a/compose/service/access_control.gen.go +++ b/compose/service/access_control.gen.go @@ -291,6 +291,11 @@ func (svc accessControl) List() (out []map[string]string) { "any": types.RecordRbacResource(0, 0, 0), "op": "owner.manage", }, + { + "type": types.RecordResourceType, + "any": types.RecordRbacResource(0, 0, 0), + "op": "revisions.search", + }, { "type": types.ComponentResourceType, "any": types.ComponentRbacResource(), @@ -609,6 +614,13 @@ func (svc accessControl) CanManageOwnerOnRecord(ctx context.Context, r *types.Re return svc.can(ctx, "owner.manage", r) } +// CanSearchRevisionsOnRecord checks if current user can revisions.search +// +// This function is auto-generated +func (svc accessControl) CanSearchRevisionsOnRecord(ctx context.Context, r *types.Record) bool { + return svc.can(ctx, "revisions.search", r) +} + // CanGrant checks if current user can manage compose permissions // // This function is auto-generated @@ -790,10 +802,11 @@ func rbacResourceOperations(r string) map[string]bool { } case types.RecordResourceType: return map[string]bool{ - "read": true, - "update": true, - "delete": true, - "owner.manage": true, + "read": true, + "update": true, + "delete": true, + "owner.manage": true, + "revisions.search": true, } case types.ComponentResourceType: return map[string]bool{ diff --git a/compose/service/module.go b/compose/service/module.go index e3f965f31..f7fba0f08 100644 --- a/compose/service/module.go +++ b/compose/service/module.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "github.com/cortezaproject/corteza-server/pkg/revisions" "reflect" "sort" "strconv" @@ -79,8 +80,8 @@ type ( SearchModelIssues(connectionID, ID uint64) []error } - dalIdentFormatter interface { - Format(ctx context.Context, template string) (out string, ok bool) + identFormatter interface { + Format(context.Context, string, ...string) (string, bool) } ) @@ -101,6 +102,7 @@ const ( sysID = "ID" sysNamespaceID = "namespaceID" sysModuleID = "moduleID" + sysRevision = "revision" sysCreatedAt = "createdAt" sysCreatedBy = "createdBy" sysUpdatedAt = "updatedAt" @@ -112,6 +114,7 @@ const ( colSysID = "id" colSysNamespaceID = "rel_namespace" colSysModuleID = "module_id" + colSysRevision = "revision" colSysCreatedAt = "created_at" colSysCreatedBy = "created_by" colSysUpdatedAt = "updated_at" @@ -125,6 +128,7 @@ var ( systemFields = slice.ToStringBoolMap([]string{ "recordID", "ownedBy", + "revision", "createdBy", "createdAt", "updatedBy", @@ -1113,6 +1117,7 @@ func modulesForNamespace(ns *types.Namespace, mm types.ModuleSet) (out types.Mod out = append(out, m) } } + return } @@ -1172,7 +1177,6 @@ func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace var ( cm dal.ConnectionMeta attrAux dal.AttributeSet - ok bool ) for connectionID, modules := range modulesByConnection(modules...) { @@ -1204,21 +1208,13 @@ func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace Capabilities: mod.Config.DAL.Capabilities, } - // - make the model ident - ident := cm.DefaultModelIdent + model.Ident = cm.DefaultModelIdent if mod.Config.DAL.Partitioned { - tpl := mod.Config.DAL.PartitionFormat - if tpl == "" { - tpl = cm.DefaultPartitionFormat - } - - ident, ok = ff.Format(ctx, tpl, formatterModuleParams(mod)...) - if !ok { - err = fmt.Errorf("invalid model ident generated: %s", ident) + model.Ident, err = makeModelIdent(ctx, ff, cm, mod, mod.Config.DAL.PartitionFormat) + if err != nil { return } } - model.Ident = ident // Convert user-defined fields to attributes attrAux, err = moduleFieldsToAttributes(ctx, cm, ns, mod) @@ -1235,12 +1231,53 @@ func moduleToModel(ctx context.Context, dmm dalModelManager, ns *types.Namespace model.Attributes = append(model.Attributes, attrAux...) out = append(out, model) + + if mod.Config.RecordRevisions.Enabled { + rModel := revisions.Model() + + // reuse the connection from the module + rModel.ConnectionID = connectionID + rModel.Resource = model.Resource + + // a little trick to ensure revision model keeps t + // the same ID and to avoid collisions with the model + rModel.ResourceID = mod.ID + 1 + + if mod.Config.RecordRevisions.Ident == "" { + mod.Config.RecordRevisions.Ident = "compose_record_revisions" + } + + rModel.Ident, err = makeModelIdent(ctx, ff, cm, mod, mod.Config.RecordRevisions.Ident) + if err != nil { + return + } + + out = append(out, rModel) + } } } return } +func makeModelIdent(ctx context.Context, ff identFormatter, cm dal.ConnectionMeta, mod *types.Module, ident string) (_ string, err error) { + var ( + ok bool + ) + + if ident == "" { + ident = cm.DefaultPartitionFormat + } + + ident, ok = ff.Format(ctx, ident, formatterModuleParams(mod)...) + if !ok { + err = fmt.Errorf("invalid model ident generated: %s", ident) + return + } + + return ident, nil +} + // moduleFieldsToAttributes converts all user-defined module fields to attributes func moduleFieldsToAttributes(ctx context.Context, cm dal.ConnectionMeta, ns *types.Namespace, mod *types.Module) (out dal.AttributeSet, err error) { out = make(dal.AttributeSet, 0, len(mod.Fields)) @@ -1295,6 +1332,10 @@ func partitionedModuleSystemFieldsToAttributes(cm dal.ConnectionMeta, mod *types out = append(out, dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, modelFieldCodec(cm, mod, mf(sysNamespaceID, sysEnc.NamespaceID)))) } + if sysEnc.Revision != nil { + out = append(out, dal.FullAttribute(sysRevision, &dal.TypeID{}, modelFieldCodec(cm, mod, mf(sysRevision, sysEnc.Revision)))) + } + if sysEnc.OwnedBy != nil { out = append(out, dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, modelFieldCodec(cm, mod, mf(sysOwnedBy, sysEnc.OwnedBy)))) } @@ -1335,6 +1376,8 @@ func defaultModuleSystemFieldsToAttributes() dal.AttributeSet { dal.FullAttribute(sysModuleID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysModuleID}), dal.FullAttribute(sysNamespaceID, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysNamespaceID}), + dal.FullAttribute(sysRevision, &dal.TypeNumber{}, &dal.CodecAlias{Ident: colSysRevision}), + dal.FullAttribute(sysOwnedBy, &dal.TypeID{}, &dal.CodecAlias{Ident: colSysOwnedBy}), dal.FullAttribute(sysCreatedAt, &dal.TypeTimestamp{}, &dal.CodecAlias{Ident: colSysCreatedAt}), diff --git a/compose/service/record.go b/compose/service/record.go index 7d614af6d..8a9b67153 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/cortezaproject/corteza-server/pkg/revisions" "regexp" "sort" "strconv" @@ -47,6 +48,8 @@ type ( namespace namespaceFinder module moduleFinder + revisions *recordRevisions + formatter recordValuesFormatter sanitizer recordValuesSanitizer validator recordValuesValidator @@ -86,6 +89,7 @@ type ( CanReadRecord(context.Context, *types.Record) bool CanUpdateRecord(context.Context, *types.Record) bool CanDeleteRecord(context.Context, *types.Record) bool + CanSearchRevisionsOnRecord(context.Context, *types.Record) bool recordManageOwnerAccessController recordValueAccessController @@ -105,6 +109,7 @@ type ( Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error) Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error) + SearchRevisions(ctx context.Context, namespaceID, moduleID, recordID uint64) (dal.Iterator, error) RecordExport(context.Context, types.RecordFilter) error RecordImport(context.Context, error) error @@ -177,6 +182,8 @@ func Record() *record { namespace: DefaultNamespace, module: DefaultModule, + revisions: &recordRevisions{revisions.Service(dal.Service())}, + formatter: values.Formatter(), sanitizer: values.Sanitizer(), } @@ -435,6 +442,43 @@ func (svc record) findSensitive(ctx context.Context, userID uint64, namespace *t return } +// SearchRevisions returns iterator for revisions of a record +// +func (svc record) SearchRevisions(ctx context.Context, namespaceID, moduleID, recordID uint64) (dal.Iterator, error) { + var ( + aProps = &recordActionProps{record: &types.Record{NamespaceID: namespaceID, ModuleID: moduleID, ID: recordID}} + + rec *types.Record + iter dal.Iterator + mod *types.Module + ns *types.Namespace + ) + + err := func() (err error) { + ns, mod, rec, err = loadRecordCombo(ctx, svc.store, svc.dal, namespaceID, moduleID, recordID) + if err != nil { + return + } + + aProps.setModule(mod) + aProps.setNamespace(ns) + aProps.setRecord(rec) + + if !mod.Config.RecordRevisions.Enabled { + return RecordErrRevisionsDisabledOnModule() + } + + if !svc.ac.CanSearchRevisionsOnRecord(ctx, rec) { + return RecordErrNotAllowedToSearchRevisions() + } + + iter, err = svc.revisions.search(ctx, rec) + return + }() + + return iter, svc.recordAction(ctx, aProps, RecordActionSearchRevisions, err) +} + func (svc record) RecordImport(ctx context.Context, err error) error { return svc.recordAction(ctx, &recordActionProps{}, RecordActionImport, err) } @@ -617,6 +661,13 @@ func (svc record) create(ctx context.Context, new *types.Record) (rec *types.Rec return } + // store revision + if m.Config.RecordRevisions.Enabled { + if err = svc.revisions.created(ctx, new); err != nil { + return + } + } + if err = label.Create(ctx, svc.store, new); err != nil { return } @@ -909,7 +960,18 @@ func (svc record) update(ctx context.Context, upd *types.Record) (rec *types.Rec } } - return dalutils.ComposeRecordUpdate(ctx, svc.dal, m, upd) + if err = dalutils.ComposeRecordUpdate(ctx, svc.dal, m, upd); err != nil { + return err + } + + if m.Config.RecordRevisions.Enabled { + // Prepare record revision for update + if err = svc.revisions.updated(ctx, upd, old); err != nil { + return err + } + } + + return nil }) if err != nil { @@ -982,6 +1044,7 @@ func (svc record) procCreate(ctx context.Context, invokerID uint64, m *types.Mod // Reset values to new record // to make sure nobody slips in something we do not want new.ID = nextID() + new.Revision = 1 new.CreatedBy = invokerID new.CreatedAt = *nowUTC() new.UpdatedAt = nil @@ -1023,6 +1086,8 @@ func (svc record) Update(ctx context.Context, upd *types.Record) (rec *types.Rec // // Both these points introduce external data that need to be checked fully in the same manner func (svc record) procUpdate(ctx context.Context, invokerID uint64, m *types.Module, upd *types.Record, old *types.Record) (rve *types.RecordValueErrorSet) { + upd.Revision = old.Revision + 1 + // Mark all values as updated (new) upd.Values.SetUpdatedFlag(true) @@ -1096,12 +1161,25 @@ func (svc record) delete(ctx context.Context, namespaceID, moduleID, recordID ui // ensure module ref is set before running through records workflows and scripts del.SetModule(m) + // deleted, revision need to be set when RecordBeforeDelete is triggered + del.DeletedAt = nowUTC() + del.DeletedBy = invokerID + del.Revision = del.Revision + 1 + { // Calling before-record-delete scripts if err = svc.eventbus.WaitFor(ctx, event.RecordBeforeDelete(nil, del, m, ns, nil, nil)); err != nil { return nil, err } } + + if m.Config.RecordRevisions.Enabled { + // Prepare record revision for update + if err = svc.revisions.softDeleted(ctx, del); err != nil { + return + } + } + if err = dalutils.ComposeRecordSoftDelete(ctx, svc.dal, m, del); err != nil { return nil, err } diff --git a/compose/service/record_actions.gen.go b/compose/service/record_actions.gen.go index 0c8d0e357..aa1ced423 100644 --- a/compose/service/record_actions.gen.go +++ b/compose/service/record_actions.gen.go @@ -612,6 +612,26 @@ func RecordActionImport(props ...*recordActionProps) *recordAction { return a } +// RecordActionSearchRevisions returns "compose:record.searchRevisions" action +// +// This function is auto-generated. +// +func RecordActionSearchRevisions(props ...*recordActionProps) *recordAction { + a := &recordAction{ + timestamp: time.Now(), + resource: "compose:record", + action: "searchRevisions", + log: "record revisions searched", + severity: actionlog.Notice, + } + + if len(props) > 0 { + a.props = props[0] + } + + return a +} + // RecordActionExport returns "compose:record.export" action // // This function is auto-generated. @@ -1102,6 +1122,78 @@ func RecordErrNotAllowedToSearch(mm ...*recordActionProps) *errors.Error { return e } +// RecordErrNotAllowedToSearchRevisions returns "compose:record.notAllowedToSearchRevisions" as *errors.Error +// +// +// This function is auto-generated. +// +func RecordErrNotAllowedToSearchRevisions(mm ...*recordActionProps) *errors.Error { + var p = &recordActionProps{} + if len(mm) > 0 { + p = mm[0] + } + + var e = errors.New( + errors.KindInternal, + + p.Format("not allowed to search or list record revisions", nil), + + errors.Meta("type", "notAllowedToSearchRevisions"), + errors.Meta("resource", "compose:record"), + + // action log entry; no formatting, it will be applied inside recordAction fn. + errors.Meta(recordLogMetaKey{}, "failed to search or list record revisions; insufficient permissions"), + errors.Meta(recordPropsMetaKey{}, p), + + // translation namespace & key + errors.Meta(locale.ErrorMetaNamespace{}, "compose"), + errors.Meta(locale.ErrorMetaKey{}, "record.errors.notAllowedToSearchRevisions"), + + errors.StackSkip(1), + ) + + if len(mm) > 0 { + } + + return e +} + +// RecordErrRevisionsDisabledOnModule returns "compose:record.revisionsDisabledOnModule" as *errors.Error +// +// +// This function is auto-generated. +// +func RecordErrRevisionsDisabledOnModule(mm ...*recordActionProps) *errors.Error { + var p = &recordActionProps{} + if len(mm) > 0 { + p = mm[0] + } + + var e = errors.New( + errors.KindInternal, + + p.Format("revisions are disabled on module", nil), + + errors.Meta("type", "revisionsDisabledOnModule"), + errors.Meta("resource", "compose:record"), + + // action log entry; no formatting, it will be applied inside recordAction fn. + errors.Meta(recordLogMetaKey{}, "failed to search or list record revisions; disabled on module"), + errors.Meta(recordPropsMetaKey{}, p), + + // translation namespace & key + errors.Meta(locale.ErrorMetaNamespace{}, "compose"), + errors.Meta(locale.ErrorMetaKey{}, "record.errors.revisionsDisabledOnModule"), + + errors.StackSkip(1), + ) + + if len(mm) > 0 { + } + + return e +} + // RecordErrNotAllowedToReadNamespace returns "compose:record.notAllowedToReadNamespace" as *errors.Error // // diff --git a/compose/service/record_actions.yaml b/compose/service/record_actions.yaml index 3411806e0..52499b2b4 100644 --- a/compose/service/record_actions.yaml +++ b/compose/service/record_actions.yaml @@ -72,6 +72,9 @@ actions: - action: import log: "records imported" + - action: searchRevisions + log: "record revisions searched" + - action: export log: "records exported" @@ -130,6 +133,14 @@ errors: message: "not allowed to search or list records" log: "failed to search or list records; insufficient permissions" + - error: notAllowedToSearchRevisions + message: "not allowed to search or list record revisions" + log: "failed to search or list record revisions; insufficient permissions" + + - error: revisionsDisabledOnModule + message: "revisions are disabled on module" + log: "failed to search or list record revisions; disabled on module" + - error: notAllowedToReadNamespace message: "not allowed to read this namespace" log: "failed to read namespace {{namespace}}; insufficient permissions" diff --git a/compose/service/record_revisions.go b/compose/service/record_revisions.go new file mode 100644 index 000000000..4d1096baf --- /dev/null +++ b/compose/service/record_revisions.go @@ -0,0 +1,144 @@ +package service + +import ( + "context" + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/auth" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/revisions" +) + +type ( + // wrapper around revision.service + // + // handles all module-to-model conversion + recordRevisions struct { + r interface { + Search(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error) + Create(ctx context.Context, mf dal.ModelRef, revision *revisions.Revision) error + } + } +) + +// constructs DAL model-ref from module +// +// Since we're using revisions model resource type needs to be altered. +func (svc *recordRevisions) modelRef(mod *types.Module) (mf dal.ModelRef) { + mf = mod.ModelRef() + + // removing resource ID from model filter + // this will force DAL service to use FindModelByResourceIdent + // to find model with same Resource value and RevisionResourceType + // for ResourceType + mf.ResourceID = 0 + mf.ResourceType = revisions.RevisionResourceType + return +} + +func (svc *recordRevisions) search(ctx context.Context, rec *types.Record) (_ dal.Iterator, err error) { + return svc.r.Search( + ctx, + svc.modelRef(rec.GetModule()), + filter.Generic(filter.WithConstraint("rel_resource", rec.ID)), + ) +} + +func (svc *recordRevisions) created(ctx context.Context, new *types.Record) (err error) { + var ( + skipList = svc.skippedField(new.GetModule()) + invokerID = auth.GetIdentityFromContext(ctx).Identity() + rev *revisions.Revision + ) + + rev = revisions.Make(revisions.Created, new.Revision, new.ID, invokerID) + err = rev.CollectChanges(new, nil, skipList...) + if err != nil { + return + } + + return svc.r.Create(ctx, svc.modelRef(new.GetModule()), rev) +} + +func (svc *recordRevisions) updated(ctx context.Context, upd, old *types.Record) (err error) { + var ( + skipList = svc.skippedField(old.GetModule()) + invokerID = auth.GetIdentityFromContext(ctx).Identity() + rev *revisions.Revision + ) + + rev = revisions.Make(revisions.Updated, upd.Revision, upd.ID, invokerID) + err = rev.CollectChanges(upd, old, skipList...) + if err != nil { + return + } + + return svc.r.Create(ctx, svc.modelRef(upd.GetModule()), rev) +} + +func (svc *recordRevisions) softDeleted(ctx context.Context, del *types.Record) (err error) { + var ( + invokerID = auth.GetIdentityFromContext(ctx).Identity() + rev *revisions.Revision + ) + + rev = revisions.Make(revisions.SoftDeleted, del.Revision, del.ID, invokerID) + if err != nil { + return + } + + return svc.r.Create(ctx, svc.modelRef(del.GetModule()), rev) +} + +func (svc *recordRevisions) skippedField(mod *types.Module) []string { + list := []string{ + "ID", + "namespaceID", + "moduleID", + "revision", + "createdBy", + "createdAt", + "updatedBy", + "updatedAt", + "deletedBy", + "deletedAt", + } + + for _, f := range mod.Fields { + if f.Config.RecordRevisions.Skip { + list = append(list, f.Name) + } + } + + return list +} + +// @todo uncomment when supported +//func (svc *recordRevisions) restored(ctx context.Context, del *types.Record) (err error) { +// var ( +// invokerID = auth.GetIdentityFromContext(ctx).Identity() +// rev *revisions.Revision +// ) +// +// rev, err = revisions.Make(revisions.Restored, del.Revision, del.ID, invokerID, nil, nil) +// if err != nil { +// return +// } +// +// return svc.r.Create(ctx, svc.modelRef(del.GetModule()), rev) +//} + +// @todo uncomment when supported +//func (svc *recordRevisions) hardDeleted(ctx context.Context, del *types.Record) (err error) { +// var ( +// invokerID = auth.GetIdentityFromContext(ctx).Identity() +// rev *revisions.Revision +// ) +// +// rev, err = revisions.Make(revisions.HardDeleted, del.Revision, del.ID, invokerID, nil, nil) +// if err != nil { +// return +// } +// +// return svc.r.Create(ctx, svc.modelRef(del.GetModule()), rev) +//} diff --git a/compose/service/record_revisions_test.go b/compose/service/record_revisions_test.go new file mode 100644 index 000000000..1fa7bce21 --- /dev/null +++ b/compose/service/record_revisions_test.go @@ -0,0 +1,63 @@ +package service + +import ( + "context" + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/dal" + "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/revisions" + "github.com/stretchr/testify/require" + "testing" +) + +type ( + mockRecordRevisionsDAL struct { + search func(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error) + create func(ctx context.Context, mf dal.ModelRef, revision *revisions.Revision) error + } +) + +func (svc *mockRecordRevisionsDAL) Search(ctx context.Context, mf dal.ModelRef, f filter.Filter) (_ dal.Iterator, err error) { + return svc.search(ctx, mf, f) +} + +func (svc *mockRecordRevisionsDAL) Create(ctx context.Context, mf dal.ModelRef, rev *revisions.Revision) error { + return svc.create(ctx, mf, rev) +} + +func TestRecordRevisions(t *testing.T) { + var ( + req = require.New(t) + ctx = context.Background() + skip = types.ModuleFieldConfig{RecordRevisions: types.ModuleFieldConfigRecordRevisions{Skip: true}} + mod = &types.Module{ + ID: 1, + Name: "test", + Fields: []*types.ModuleField{ + {Name: "rev1", Kind: "string"}, + {Name: "rev2", Kind: "string"}, + {Name: "rev3", Kind: "string", Config: skip}, + }, + } + + rec = &types.Record{ID: 2, ModuleID: 1} + + dalMock = &mockRecordRevisionsDAL{} + + svc = &recordRevisions{r: dalMock} + ) + + rec.SetModule(mod) + req.NoError(rec.SetValue("rev1", 0, "val1")) + req.NoError(rec.SetValue("rev2", 0, "val2")) + req.NoError(rec.SetValue("rev3", 0, "val3")) + + var changes int + dalMock.create = func(ctx context.Context, mf dal.ModelRef, r *revisions.Revision) error { + changes = len(r.Changes) + return nil + } + + req.NoError(svc.created(ctx, rec)) + req.Equal(3, changes, "expecting ownedBy, rev1 and rev2") +} diff --git a/compose/service/record_test.go b/compose/service/record_test.go index af80f8ff9..4006a3b40 100644 --- a/compose/service/record_test.go +++ b/compose/service/record_test.go @@ -507,8 +507,8 @@ func TestRecord_refAccessControl(t *testing.T) { req = require.New(t) // uncomment to enable sql conn debugging - //ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) - ctx = context.Background() + ctx = logger.ContextWithValue(context.Background(), logger.MakeDebugLogger()) + //ctx = context.Background() s, err = sqlite.ConnectInMemoryWithDebug(ctx) ) diff --git a/compose/types/module.go b/compose/types/module.go index 8042d46c4..d72e7836e 100644 --- a/compose/types/module.go +++ b/compose/types/module.go @@ -51,6 +51,8 @@ type ( ModuleID *EncodingStrategy `json:"moduleID"` NamespaceID *EncodingStrategy `json:"namespaceID"` + Revision *EncodingStrategy `json:"revision"` + OwnedBy *EncodingStrategy `json:"ownedBy"` CreatedAt *EncodingStrategy `json:"createdAt"` @@ -72,6 +74,8 @@ type ( // @todo we need to transfer this from meta!! Discovery discovery.ModuleMeta `json:"discovery"` + + RecordRevisions ModuleConfigRecordRevisions `json:"recordRevisions"` } ModuleConfigDAL struct { @@ -88,6 +92,14 @@ type ( SystemFieldEncoding SystemFieldEncoding `json:"systemFieldEncoding"` } + ModuleConfigRecordRevisions struct { + // enable or disable revisions + Enabled bool `json:"enabled"` + + // where are record revisions stored + Ident string `json:"ident"` + } + ModuleConfigDataPrivacy struct { // Define the highest sensitivity level which // can be configured on the module fields diff --git a/compose/types/module_field.go b/compose/types/module_field.go index fde7ed636..6beb1a735 100644 --- a/compose/types/module_field.go +++ b/compose/types/module_field.go @@ -53,6 +53,8 @@ type ( ModuleFieldConfig struct { DAL ModuleFieldConfigDAL `json:"dal"` Privacy ModuleFieldConfigDataPrivacy `json:"privacy"` + + RecordRevisions ModuleFieldConfigRecordRevisions `json:"recordRevisions"` } ModuleFieldConfigDAL struct { @@ -67,6 +69,11 @@ type ( UsageDisclosure string `json:"usageDisclosure"` } + ModuleFieldConfigRecordRevisions struct { + // when true, skip record revisions for this field + Skip bool `json:"enabled"` + } + EncodingStrategy struct { *EncodingStrategyAlias `json:"alias,omitempty"` *EncodingStrategyJSON `json:"json,omitempty"` diff --git a/compose/types/record.go b/compose/types/record.go index aaf666fa2..98a8caf4b 100644 --- a/compose/types/record.go +++ b/compose/types/record.go @@ -37,6 +37,8 @@ type ( ID uint64 `json:"recordID,string"` ModuleID uint64 `json:"moduleID,string"` + Revision uint `json:"revision,omitempty"` + module *Module Values RecordValueSet `json:"values,omitempty"` @@ -49,9 +51,9 @@ type ( CreatedAt time.Time `json:"createdAt,omitempty"` CreatedBy uint64 `json:"createdBy,string" ` UpdatedAt *time.Time `json:"updatedAt,omitempty"` - UpdatedBy uint64 `json:"updatedBy,string,omitempty" ` + UpdatedBy uint64 `json:"updatedBy,string,omitempty"` DeletedAt *time.Time `json:"deletedAt,omitempty"` - DeletedBy uint64 `json:"deletedBy,string,omitempty" ` + DeletedBy uint64 `json:"deletedBy,string,omitempty"` } RecordFilter struct { @@ -193,6 +195,8 @@ func (r *Record) GetValue(name string, pos uint) (any, error) { return r.ModuleID, nil case "namespaceID": return r.NamespaceID, nil + case "revision": + return r.Revision, nil case "createdAt": return r.CreatedAt, nil case "createdBy": @@ -217,33 +221,40 @@ func (r *Record) GetValue(name string, pos uint) (any, error) { } // CountValues returns how many values per field are there -func (r *Record) CountValues() map[string]uint { +func (r *Record) CountValues() (pos map[string]uint) { var ( - pos = map[string]uint{ - "ID": 1, - "moduleID": 1, - "namespaceID": 1, - "createdAt": 1, - "createdBy": 1, - "updatedAt": 1, - "updatedBy": 1, - "deletedAt": 1, - "deletedBy": 1, - "ownedBy": 1, - } + mod = r.GetModule() ) - if mod := r.GetModule(); mod != nil { - for _, f := range mod.Fields { - pos[f.Name] = 0 - } + pos = map[string]uint{ + "ID": 1, + "moduleID": 1, + "namespaceID": 1, + "revision": 1, + "createdAt": 1, + "createdBy": 1, + "updatedAt": 1, + "updatedBy": 1, + "deletedAt": 1, + "deletedBy": 1, + "ownedBy": 1, + } + + if mod == nil { + // count record values + // only when module is known + return + } + + for _, f := range mod.Fields { + pos[f.Name] = 0 } for _, val := range r.Values { pos[val.Name]++ } - return pos + return } func (r *Record) SetValue(name string, pos uint, value any) (err error) { @@ -262,6 +273,8 @@ func (r *Record) SetValue(name string, pos uint, value any) (err error) { return cast2.Uint64(value, &r.DeletedBy) case "ownedBy": return cast2.Uint64(value, &r.OwnedBy) + case "revision": + return cast2.Uint(value, &r.Revision) case "createdAt": return cast2.Time(value, &r.CreatedAt) case "updatedAt": @@ -323,6 +336,7 @@ func (r Record) Dict() map[string]interface{} { "ID": r.ID, "recordID": r.ID, "moduleID": r.ModuleID, + "revision": r.Revision, "labels": r.Labels, "namespaceID": r.NamespaceID, "ownedBy": r.OwnedBy, diff --git a/pkg/revisions/revisions.go b/pkg/revisions/revisions.go index 58d17daa8..aa33016bd 100644 --- a/pkg/revisions/revisions.go +++ b/pkg/revisions/revisions.go @@ -59,7 +59,7 @@ func Make(op Operation, revision uint, resourceID, userID uint64) (rev *Revision } // untested -func (r *Revision) CollectChanges(new, old dal.ValueGetter, omit ...string) (err error) { +func (r *Revision) CollectChanges(new, old dal.ValueGetter, skip ...string) (err error) { var ( cc = make([]*Change, 0) ch *Change @@ -86,8 +86,8 @@ keys: continue keys } - for _, o := range omit { - if key == o { + for _, s := range skip { + if key == s { continue keys } } diff --git a/provision/000_base/compose_access_control.yaml b/provision/000_base/compose_access_control.yaml index def8e9041..e7fea5a35 100644 --- a/provision/000_base/compose_access_control.yaml +++ b/provision/000_base/compose_access_control.yaml @@ -60,6 +60,7 @@ allow: - read - update - delete + - revisions.search corteza::compose:chart/*/*: - read diff --git a/store/adapters/rdbms/dal/iterator.go b/store/adapters/rdbms/dal/iterator.go index 2e1faa070..45a159cc7 100644 --- a/store/adapters/rdbms/dal/iterator.go +++ b/store/adapters/rdbms/dal/iterator.go @@ -161,7 +161,6 @@ func (i *iterator) fetch(ctx context.Context) (_ *sql.Rows, err error) { } } - sql, _, _ = query.Prepared(false).ToSQL() if sql, args, err = query.ToSQL(); err != nil { return nil, err } diff --git a/store/adapters/rdbms/upgrade.go b/store/adapters/rdbms/upgrade.go index db509a23c..b9767b147 100644 --- a/store/adapters/rdbms/upgrade.go +++ b/store/adapters/rdbms/upgrade.go @@ -30,6 +30,7 @@ func (s *Store) Upgrade(ctx context.Context) (err error) { fix202209_extendComposeModuleForPrivacyAndDAL, fix202209_extendComposeModuleFieldsForPrivacyAndDAL, fix202209_dropObsoleteComposeModuleFields, + fix202209_composeRecordRevisions, } for _, fix := range fixes { diff --git a/store/adapters/rdbms/upgrade_fixes.go b/store/adapters/rdbms/upgrade_fixes.go index 44eddde7b..d5109a029 100644 --- a/store/adapters/rdbms/upgrade_fixes.go +++ b/store/adapters/rdbms/upgrade_fixes.go @@ -32,3 +32,87 @@ func fix202209_dropObsoleteComposeModuleFields(ctx context.Context, s *Store) (e "is_visible", ) } + +func fix202209_composeRecordRevisions(ctx context.Context, s *Store) (err error) { + s.log(ctx).Info("extending compose_record table with revision column") + return s.SchemaAPI.AddColumn( + ctx, s.DB, + &Table{Name: "compose_record"}, + &Column{Type: ColumnType{Type: ColumnTypeInteger}, DefaultValue: "1", Name: "revision"}, + ) + + //return // will probably not need this... + //var ( + // log = s.log(ctx) + //) + // + //const ( + // envKeySkip = "UPGRADE_COMPOSE_RECORD_CHANGES_PREFILL_SKIP" + // + // tblChanges = "compose_record_changes" + // tblRecords = "compose_record" + //) + // + //if _, set := os.LookupEnv(envKeySkip); set { + // return + //} + // + //var ( + // tblRecordColumns = []any{ + // // reusing record ID for ID of the (1st record) + // // entry in the changes table + // "id", + // "id", + // "rel_namespace", "module_id", + // "values", + // "owned_by", + // "created_at", "created_by", + // "updated_at", "updated_by", + // "deleted_at", "deleted_by", + // } + // + // tblChangesColumns = []any{ + // "id", + // "rel_record", + // "rel_namespace", "rel_module", + // "values", + // "owned_by", + // "created_at", "created_by", + // "updated_at", "updated_by", + // "deleted_at", "deleted_by", + // } + // + // d = s.Dialect + // check = d.Select(exp.NewLiteralExpression("1")).From(tblChanges).Limit(1) + // count int + // + // copyAll = d. + // Insert(tblChanges). + // Cols(tblChangesColumns...). + // FromQuery(d.From(tblRecords).Select(tblRecordColumns...)) + //) + // + //if err = s.QueryOne(ctx, check, &count); err != nil { + // // exit on error or when changes table is not empty + // if !errors.IsNotFound(err) { + // return fmt.Errorf("could not check if %q is empty: %w", tblChanges, err) + // } + // + //} else if count > 0 { + // return + //} + // + //log.Warn(fmt.Sprintf("Empty %s table detected. "+ + // "Prefilling record changes table with current records, "+ + // "this might take a while, depending amount of records you have and how fast your database is. "+ + // "If you need to disable this, stop the server, set %s=1 "+ + // "and restart the server.", + // tblChanges, + // envKeySkip, + //)) + // + //spew.Dump(copyAll.ToSQL()) + // + //// make a copy of records to changes table + //return s.Exec(ctx, copyAll) +} diff --git a/store/adapters/rdbms/upgrade_tables.go b/store/adapters/rdbms/upgrade_tables.go index 840aac1f0..5fc389de3 100644 --- a/store/adapters/rdbms/upgrade_tables.go +++ b/store/adapters/rdbms/upgrade_tables.go @@ -59,6 +59,7 @@ func Tables() []*Table { tableComposeNamespace(), tableComposePage(), tableComposeRecord(), + tableComposeRecordRevisions(), tableFederationModuleShared(), tableFederationModuleExposed(), tableFederationModuleMapping(), @@ -519,6 +520,7 @@ func tableComposeRecord() *Table { ID, ColumnDef("rel_namespace", ColumnTypeIdentifier), ColumnDef("module_id", ColumnTypeIdentifier), + ColumnDef("revision", ColumnTypeInteger), ColumnDef("values", ColumnTypeJson), ColumnDef("owned_by", ColumnTypeIdentifier), CUDTimestamps, @@ -530,6 +532,21 @@ func tableComposeRecord() *Table { ) } +func tableComposeRecordRevisions() *Table { + return TableDef("compose_record_revisions", + ID, + ColumnDef("ts", ColumnTypeTimestamp), + ColumnDef("rel_resource", ColumnTypeIdentifier), + ColumnDef("revision", ColumnTypeInteger), + ColumnDef("operation", ColumnTypeText), + ColumnDef("rel_user", ColumnTypeIdentifier), + ColumnDef("delta", ColumnTypeJson), + ColumnDef("comment", ColumnTypeText), + + AddIndex("record", IColumn("rel_resource")), + ) +} + func tableFederationModuleShared() *Table { return TableDef("federation_module_shared", ID,