From 84f918a1d7d6172ce3e9e8d8382bdee201f33d6e Mon Sep 17 00:00:00 2001 From: Vivek Patel Date: Mon, 7 Mar 2022 18:20:42 +0530 Subject: [PATCH] Add meta column to resource_activity_log table Also adds resourceID filter in discovery resource routes --- compose/service/attachment_actions.gen.go | 7 +- discovery/rest.yaml | 8 ++ discovery/rest/internal/documents/compose.go | 18 ++++- discovery/rest/internal/documents/system.go | 6 +- discovery/rest/internal/feed/feed.go | 12 +-- discovery/rest/internal/feed/resource.go | 1 + discovery/rest/request/resources.go | 84 +++++++++++++++++++- discovery/rest/resources.go | 16 ++-- pkg/discovery/service.go | 5 +- pkg/discovery/types/types.go | 44 +++++++++- store/rdbms/generic_upgrades.go | 16 +++- store/rdbms/rdbms_schema.go | 1 + store/rdbms/resource_activity_log.gen.go | 3 + store/rdbms/resource_activity_log.go | 4 +- store/resource_activity_log.yaml | 2 + 15 files changed, 197 insertions(+), 30 deletions(-) diff --git a/compose/service/attachment_actions.gen.go b/compose/service/attachment_actions.gen.go index 95a85fec3..35bd5fab9 100644 --- a/compose/service/attachment_actions.gen.go +++ b/compose/service/attachment_actions.gen.go @@ -11,13 +11,12 @@ package service import ( "context" "fmt" - "strings" - "time" - "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/actionlog" "github.com/cortezaproject/corteza-server/pkg/errors" "github.com/cortezaproject/corteza-server/pkg/locale" + "strings" + "time" ) type ( @@ -940,7 +939,7 @@ func AttachmentErrNotAllowedToUploadThisType(mm ...*attachmentActionProps) *erro errors.Meta("resource", "compose:attachment"), // action log entry; no formatting, it will be applied inside recordAction fn. - errors.Meta(attachmentLogMetaKey{}, "could not upload this file, file type nto allowed"), + errors.Meta(attachmentLogMetaKey{}, "could not upload this file, file type not allowed"), errors.Meta(attachmentPropsMetaKey{}, p), // translation namespace & key diff --git a/discovery/rest.yaml b/discovery/rest.yaml index 66d2452e3..b078ad777 100644 --- a/discovery/rest.yaml +++ b/discovery/rest.yaml @@ -23,11 +23,17 @@ endpoints: method: GET title: Resource path: "/system/users" + parameters: + path: + - { name: userID, type: uint64, title: User ID } - name: composeNamespaces method: GET title: Resource path: "/compose/namespaces" + parameters: + path: + - { name: namespaceID, type: uint64, title: Namespace ID } - name: composeModules method: GET @@ -36,6 +42,7 @@ endpoints: parameters: path: - { name: namespaceID, type: uint64, title: Namespace ID } + - { name: moduleID, type: uint64, title: Module ID } - name: composeRecords method: GET @@ -45,6 +52,7 @@ endpoints: path: - { name: namespaceID, type: uint64, title: Namespace ID } - { name: moduleID, type: uint64, title: Module ID } + - { name: recordID, type: uint64, title: Record ID } - path: "/feed" entrypoint: feed diff --git a/discovery/rest/internal/documents/compose.go b/discovery/rest/internal/documents/compose.go index dccffed6b..7ae0d7792 100644 --- a/discovery/rest/internal/documents/compose.go +++ b/discovery/rest/internal/documents/compose.go @@ -81,7 +81,7 @@ func ComposeResources() *composeResources { } } -func (d composeResources) Namespaces(ctx context.Context, limit uint, cur string) (rsp *Response, err error) { +func (d composeResources) Namespaces(ctx context.Context, limit uint, cur string, namespaceID uint64) (rsp *Response, err error) { return rsp, func() (err error) { if !d.settings.Discovery.ComposeNamespaces.Enabled { return errors.Internal("compose namespace indexing disabled") @@ -97,6 +97,10 @@ func (d composeResources) Namespaces(ctx context.Context, limit uint, cur string } ) + if namespaceID > 0 { + f.NamespaceID = append(f.NamespaceID, namespaceID) + } + if f.Paging, err = filter.NewPaging(limit, cur); err != nil { return err } @@ -171,7 +175,7 @@ func (d composeResources) Namespaces(ctx context.Context, limit uint, cur string }() } -func (d composeResources) Modules(ctx context.Context, namespaceID uint64, limit uint, cur string) (rsp *Response, err error) { +func (d composeResources) Modules(ctx context.Context, namespaceID uint64, limit uint, cur string, moduleID uint64) (rsp *Response, err error) { return rsp, func() (err error) { if !d.settings.Discovery.ComposeModules.Enabled { return errors.Internal("compose module indexing disabled") @@ -186,6 +190,10 @@ func (d composeResources) Modules(ctx context.Context, namespaceID uint64, limit } ) + if moduleID > 0 { + f.ModuleID = append(f.ModuleID, moduleID) + } + if f.Paging, err = filter.NewPaging(limit, cur); err != nil { return } @@ -254,7 +262,7 @@ func (d composeResources) Modules(ctx context.Context, namespaceID uint64, limit }() } -func (d composeResources) Records(ctx context.Context, namespaceID, moduleID uint64, limit uint, cur string) (rsp *Response, err error) { +func (d composeResources) Records(ctx context.Context, namespaceID, moduleID uint64, limit uint, cur string, recordID uint64) (rsp *Response, err error) { return rsp, func() (err error) { if !d.settings.Discovery.ComposeRecords.Enabled { return errors.Internal("compose record indexing disabled") @@ -271,6 +279,10 @@ func (d composeResources) Records(ctx context.Context, namespaceID, moduleID uin } ) + if recordID > 0 { + f.Query = fmt.Sprintf("ID=%d", recordID) + } + if f.Paging, err = filter.NewPaging(limit, cur); err != nil { return err } diff --git a/discovery/rest/internal/documents/system.go b/discovery/rest/internal/documents/system.go index 97401e2e7..3f0d82cf3 100644 --- a/discovery/rest/internal/documents/system.go +++ b/discovery/rest/internal/documents/system.go @@ -41,7 +41,7 @@ func SystemResources() *systemResources { } } -func (d systemResources) Users(ctx context.Context, limit uint, cur string) (rsp *Response, err error) { +func (d systemResources) Users(ctx context.Context, limit uint, cur string, userID uint64) (rsp *Response, err error) { return rsp, func() (err error) { if !d.settings.Discovery.SystemUsers.Enabled { return errors.Internal("system user indexing disabled") @@ -54,6 +54,10 @@ func (d systemResources) Users(ctx context.Context, limit uint, cur string) (rsp } ) + if userID > 0 { + f.UserID = append(f.UserID, userID) + } + if f.Paging, err = filter.NewPaging(limit, cur); err != nil { return err } diff --git a/discovery/rest/internal/feed/feed.go b/discovery/rest/internal/feed/feed.go index 82850a013..e74211bca 100644 --- a/discovery/rest/internal/feed/feed.go +++ b/discovery/rest/internal/feed/feed.go @@ -2,6 +2,7 @@ package feed import ( "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/jmoiron/sqlx/types" "time" ) @@ -12,11 +13,12 @@ type ( } ActivityLog struct { - ID uint64 `json:"activityID,string"` - ResourceID uint64 `json:"resourceID,string"` - ResourceType string `json:"resourceType"` - ResourceAction string `json:"resourceAction"` - Timestamp time.Time `json:"timestamp"` + ID uint64 `json:"activityID,string"` + ResourceID uint64 `json:"resourceID,string"` + ResourceType string `json:"resourceType"` + ResourceAction string `json:"resourceAction"` + Timestamp time.Time `json:"timestamp"` + Meta types.JSONText `json:"meta"` } Filter struct { diff --git a/discovery/rest/internal/feed/resource.go b/discovery/rest/internal/feed/resource.go index 4c316e77e..3a3c6bb46 100644 --- a/discovery/rest/internal/feed/resource.go +++ b/discovery/rest/internal/feed/resource.go @@ -82,6 +82,7 @@ func (a resourceActivity) ResourceActivities(ctx context.Context, limit uint, cu ResourceType: a.ResourceType, ResourceAction: a.ResourceAction, Timestamp: a.Timestamp, + Meta: a.Meta, }) return nil diff --git a/discovery/rest/request/resources.go b/discovery/rest/request/resources.go index b4155d6bf..73623f494 100644 --- a/discovery/rest/request/resources.go +++ b/discovery/rest/request/resources.go @@ -34,6 +34,11 @@ var ( type ( // Internal API interface ResourcesSystemUsers struct { + // UserID PATH parameter + // + // User ID + UserID uint64 `json:",string"` + // Limit GET parameter // // Limit @@ -46,6 +51,11 @@ type ( } ResourcesComposeNamespaces struct { + // NamespaceID PATH parameter + // + // Namespace ID + NamespaceID uint64 `json:",string"` + // Limit GET parameter // // Limit @@ -63,6 +73,11 @@ type ( // Namespace ID NamespaceID uint64 `json:",string"` + // ModuleID PATH parameter + // + // Module ID + ModuleID uint64 `json:",string"` + // Limit GET parameter // // Limit @@ -85,6 +100,11 @@ type ( // Module ID ModuleID uint64 `json:",string"` + // RecordID PATH parameter + // + // Record ID + RecordID uint64 `json:",string"` + // Limit GET parameter // // Limit @@ -105,11 +125,17 @@ func NewResourcesSystemUsers() *ResourcesSystemUsers { // Auditable returns all auditable/loggable parameters func (r ResourcesSystemUsers) Auditable() map[string]interface{} { return map[string]interface{}{ + "userID": r.UserID, "limit": r.Limit, "pageCursor": r.PageCursor, } } +// Auditable returns all auditable/loggable parameters +func (r ResourcesSystemUsers) GetUserID() uint64 { + return r.UserID +} + // Auditable returns all auditable/loggable parameters func (r ResourcesSystemUsers) GetLimit() uint { return r.Limit @@ -141,6 +167,18 @@ func (r *ResourcesSystemUsers) Fill(req *http.Request) (err error) { } } + { + var val string + // path params + + val = chi.URLParam(req, "userID") + r.UserID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + + } + return err } @@ -152,11 +190,17 @@ func NewResourcesComposeNamespaces() *ResourcesComposeNamespaces { // Auditable returns all auditable/loggable parameters func (r ResourcesComposeNamespaces) Auditable() map[string]interface{} { return map[string]interface{}{ - "limit": r.Limit, - "pageCursor": r.PageCursor, + "namespaceID": r.NamespaceID, + "limit": r.Limit, + "pageCursor": r.PageCursor, } } +// Auditable returns all auditable/loggable parameters +func (r ResourcesComposeNamespaces) GetNamespaceID() uint64 { + return r.NamespaceID +} + // Auditable returns all auditable/loggable parameters func (r ResourcesComposeNamespaces) GetLimit() uint { return r.Limit @@ -188,6 +232,18 @@ func (r *ResourcesComposeNamespaces) Fill(req *http.Request) (err error) { } } + { + var val string + // path params + + val = chi.URLParam(req, "namespaceID") + r.NamespaceID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + + } + return err } @@ -200,6 +256,7 @@ func NewResourcesComposeModules() *ResourcesComposeModules { func (r ResourcesComposeModules) Auditable() map[string]interface{} { return map[string]interface{}{ "namespaceID": r.NamespaceID, + "moduleID": r.ModuleID, "limit": r.Limit, "pageCursor": r.PageCursor, } @@ -210,6 +267,11 @@ func (r ResourcesComposeModules) GetNamespaceID() uint64 { return r.NamespaceID } +// Auditable returns all auditable/loggable parameters +func (r ResourcesComposeModules) GetModuleID() uint64 { + return r.ModuleID +} + // Auditable returns all auditable/loggable parameters func (r ResourcesComposeModules) GetLimit() uint { return r.Limit @@ -251,6 +313,12 @@ func (r *ResourcesComposeModules) Fill(req *http.Request) (err error) { return err } + val = chi.URLParam(req, "moduleID") + r.ModuleID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + } return err @@ -266,6 +334,7 @@ func (r ResourcesComposeRecords) Auditable() map[string]interface{} { return map[string]interface{}{ "namespaceID": r.NamespaceID, "moduleID": r.ModuleID, + "recordID": r.RecordID, "limit": r.Limit, "pageCursor": r.PageCursor, } @@ -281,6 +350,11 @@ func (r ResourcesComposeRecords) GetModuleID() uint64 { return r.ModuleID } +// Auditable returns all auditable/loggable parameters +func (r ResourcesComposeRecords) GetRecordID() uint64 { + return r.RecordID +} + // Auditable returns all auditable/loggable parameters func (r ResourcesComposeRecords) GetLimit() uint { return r.Limit @@ -328,6 +402,12 @@ func (r *ResourcesComposeRecords) Fill(req *http.Request) (err error) { return err } + val = chi.URLParam(req, "recordID") + r.RecordID, err = payload.ParseUint64(val), nil + if err != nil { + return err + } + } return err diff --git a/discovery/rest/resources.go b/discovery/rest/resources.go index d14ee3cc5..e6cccc838 100644 --- a/discovery/rest/resources.go +++ b/discovery/rest/resources.go @@ -9,13 +9,13 @@ import ( type ( resources struct { sys interface { - Users(ctx context.Context, limit uint, cur string) (*documents.Response, error) + Users(ctx context.Context, limit uint, cur string, userID uint64) (*documents.Response, error) } cmp interface { - Namespaces(ctx context.Context, limit uint, cur string) (*documents.Response, error) - Modules(ctx context.Context, namespaceID uint64, limit uint, cur string) (*documents.Response, error) - Records(ctx context.Context, namespaceID, moduleID uint64, limit uint, cur string) (*documents.Response, error) + Namespaces(ctx context.Context, limit uint, cur string, namespaceID uint64) (*documents.Response, error) + Modules(ctx context.Context, namespaceID uint64, limit uint, cur string, moduleID uint64) (*documents.Response, error) + Records(ctx context.Context, namespaceID, moduleID uint64, limit uint, cur string, recordID uint64) (*documents.Response, error) } } ) @@ -28,17 +28,17 @@ func Resources() *resources { } func (ctrl resources) SystemUsers(ctx context.Context, r *request.ResourcesSystemUsers) (interface{}, error) { - return ctrl.sys.Users(ctx, r.Limit, r.PageCursor) + return ctrl.sys.Users(ctx, r.Limit, r.PageCursor, r.UserID) } func (ctrl resources) ComposeNamespaces(ctx context.Context, r *request.ResourcesComposeNamespaces) (interface{}, error) { - return ctrl.cmp.Namespaces(ctx, r.Limit, r.PageCursor) + return ctrl.cmp.Namespaces(ctx, r.Limit, r.PageCursor, r.NamespaceID) } func (ctrl resources) ComposeModules(ctx context.Context, r *request.ResourcesComposeModules) (interface{}, error) { - return ctrl.cmp.Modules(ctx, r.NamespaceID, r.Limit, r.PageCursor) + return ctrl.cmp.Modules(ctx, r.NamespaceID, r.Limit, r.PageCursor, r.ModuleID) } func (ctrl resources) ComposeRecords(ctx context.Context, r *request.ResourcesComposeRecords) (interface{}, error) { - return ctrl.cmp.Records(ctx, r.NamespaceID, r.ModuleID, r.Limit, r.PageCursor) + return ctrl.cmp.Records(ctx, r.NamespaceID, r.ModuleID, r.Limit, r.PageCursor, r.RecordID) } diff --git a/pkg/discovery/service.go b/pkg/discovery/service.go index 90a8a6f4b..ea843e1da 100644 --- a/pkg/discovery/service.go +++ b/pkg/discovery/service.go @@ -80,7 +80,10 @@ func (svc service) InitResourceActivityLog(ctx context.Context, resourceType []s var a *types.ResourceActivity dec, is := ev.(types.ResDecoder) if is { - svc.logger.Debug(fmt.Sprintf("resource changed, updating ActivityLog for EventType: %s and ResourceType: %s", ev.EventType(), ev.ResourceType())) + svc.logger.Debug("resource changed", + zap.String("eventType", ev.EventType()), + zap.String("resourceType", ev.ResourceType()), + ) a, err = types.CastToResourceActivity(dec) if err != nil { diff --git a/pkg/discovery/types/types.go b/pkg/discovery/types/types.go index 84c49b56e..4331c324d 100644 --- a/pkg/discovery/types/types.go +++ b/pkg/discovery/types/types.go @@ -1,11 +1,13 @@ package types import ( + "encoding/json" "fmt" composeTypes "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/pkg/id" systemTypes "github.com/cortezaproject/corteza-server/system/types" + "github.com/jmoiron/sqlx/types" "time" ) @@ -28,6 +30,9 @@ type ( // Timestamp of the raised event Timestamp time.Time `json:"timestamp"` + + // Meta of the related resources + Meta types.JSONText `json:"meta"` } ResourceActivityFilter struct { @@ -47,6 +52,11 @@ type ( filter.Paging } + ResourceActivityMeta struct { + NamespaceID uint64 `json:"namespaceID,string"` + ModuleID uint64 `json:"moduleID,string"` + } + ResDecoder interface { EventType() string ResourceType() string @@ -99,6 +109,22 @@ func CastToResourceActivity(dec ResDecoder) (a *ResourceActivity, err error) { setResourceID := func(ID uint64) { a.ResourceID = ID } + setMeta := func(nsID, mID uint64) error { + var meta ResourceActivityMeta + if nsID > 0 { + meta.NamespaceID = nsID + } + if mID > 0 { + meta.ModuleID = mID + } + + a.Meta, err = json.Marshal(meta) + if err != nil { + return err + } + + return nil + } switch a.ResourceType { case "system:user": // @todo system/service/service.go#134 @@ -115,14 +141,24 @@ func CastToResourceActivity(dec ResDecoder) (a *ResourceActivity, err error) { } case (composeTypes.Module{}).LabelResourceKind(): if v, ok := dec.(mDecoder); ok { - if v.Module() != nil { - setResourceID(v.Module().ID) + mod := v.Module() + if mod != nil { + setResourceID(mod.ID) + err = setMeta(mod.NamespaceID, 0) + if err != nil { + return + } } } case (composeTypes.Record{}).LabelResourceKind(): if v, ok := dec.(recDecoder); ok { - if v.Record() != nil { - setResourceID(v.Record().ID) + rec := v.Record() + if rec != nil { + setResourceID(rec.ID) + err = setMeta(rec.NamespaceID, rec.ModuleID) + if err != nil { + return + } } } default: diff --git a/store/rdbms/generic_upgrades.go b/store/rdbms/generic_upgrades.go index bd12a131d..07cd40799 100644 --- a/store/rdbms/generic_upgrades.go +++ b/store/rdbms/generic_upgrades.go @@ -99,7 +99,10 @@ func (g genericUpgrades) Upgrade(ctx context.Context, t *ddl.Table) error { return g.all(ctx, g.AddScenariosField, ) - + case "resource_activity_log": + return g.all(ctx, + g.AddResourceActivityLogMetaField, + ) } return nil @@ -454,3 +457,14 @@ func (g genericUpgrades) CreateAutomationSessionIndexes(ctx context.Context) (er return } + +func (g genericUpgrades) AddResourceActivityLogMetaField(ctx context.Context) error { + _, err := g.u.AddColumn(ctx, "resource_activity_log", &ddl.Column{ + Name: "meta", + Type: ddl.ColumnType{Type: ddl.ColumnTypeJson}, + IsNull: false, + DefaultValue: "'{}'", + }) + + return err +} diff --git a/store/rdbms/rdbms_schema.go b/store/rdbms/rdbms_schema.go index 74a7d7a17..855a3af54 100644 --- a/store/rdbms/rdbms_schema.go +++ b/store/rdbms/rdbms_schema.go @@ -750,6 +750,7 @@ func (Schema) ResourceActivityLog() *Table { ColumnDef("resource_type", ColumnTypeText, ColumnTypeLength(handleLength)), ColumnDef("resource_action", ColumnTypeVarchar, ColumnTypeLength(handleLength)), ColumnDef("ts", ColumnTypeTimestamp), + ColumnDef("meta", ColumnTypeJson), AddIndex("rel_resource", IColumn("rel_resource")), AddIndex("ts", IColumn("ts")), diff --git a/store/rdbms/resource_activity_log.gen.go b/store/rdbms/resource_activity_log.gen.go index 0c74c606d..823ac6d52 100644 --- a/store/rdbms/resource_activity_log.gen.go +++ b/store/rdbms/resource_activity_log.gen.go @@ -434,6 +434,7 @@ func (s Store) internalResourceActivityLogRowScanner(row rowScanner) (res *types &res.ResourceType, &res.ResourceAction, &res.Timestamp, + &res.Meta, ) if err == sql.ErrNoRows { @@ -477,6 +478,7 @@ func (Store) resourceActivityLogColumns(aa ...string) []string { alias + "resource_type", alias + "resource_action", alias + "ts", + alias + "meta", } } @@ -502,6 +504,7 @@ func (s Store) internalResourceActivityLogEncoder(res *types.ResourceActivity) s "resource_type": res.ResourceType, "resource_action": res.ResourceAction, "ts": res.Timestamp, + "meta": res.Meta, } } diff --git a/store/rdbms/resource_activity_log.go b/store/rdbms/resource_activity_log.go index 50fbf5ec5..0ab6b496f 100644 --- a/store/rdbms/resource_activity_log.go +++ b/store/rdbms/resource_activity_log.go @@ -7,7 +7,7 @@ import ( ) func (s Store) convertResourceActivityLogFilter(f types.ResourceActivityFilter) (query squirrel.SelectBuilder, err error) { - query = s.actionlogsSelectBuilder() + query = s.resourceActivityLogsSelectBuilder() // Always sort by ID descending query = query.OrderBy("id DESC") @@ -36,6 +36,7 @@ func (s Store) scanResourceActivityLogRow(row rowScanner, res *types.ResourceAct &res.ResourceType, &res.ResourceAction, &res.Timestamp, + &res.Meta, ) if err != nil { @@ -53,6 +54,7 @@ func (s Store) encodeResourceActivityLog(res *types.ResourceActivity) store.Payl "resource_type": res.ResourceType, "resource_action": res.ResourceAction, "ts": res.Timestamp, + "meta": res.Meta, } return out diff --git a/store/resource_activity_log.yaml b/store/resource_activity_log.yaml index 4efab7ec7..45e593d46 100644 --- a/store/resource_activity_log.yaml +++ b/store/resource_activity_log.yaml @@ -11,6 +11,7 @@ fields: - { field: ResourceType } - { field: ResourceAction } - { field: Timestamp, type: "time.Time" } + - { field: Meta, type: "types.JSONText" } lookups: - fields: [ ID ] @@ -24,6 +25,7 @@ rdbms: table: resource_activity_log mapFields: Timestamp: { column: ts } + Meta: { column: meta } create: export: true