diff --git a/compose/rest.yaml b/compose/rest.yaml index b286875ff..88eecc61f 100644 --- a/compose/rest.yaml +++ b/compose/rest.yaml @@ -986,6 +986,16 @@ endpoints: get: - { name: sensitivityLevelID, type: "uint64", title: "Sensitivity Level ID", required: false} - { name: connectionID, type: "[]string", required: false, title: "Filter by connection ID"} + - name: module list + method: GET + title: List modules + path: /module + parameters: + get: + - { name: connectionID, type: "[]string", title: "Filter by connection ID" } + - { name: limit, type: "uint", title: "Limit" } + - { name: pageCursor, type: "string", title: "Page cursor" } + - { name: sort, type: "string", title: "Sort items" } - title: Charts path: "/namespace/{namespaceID}/chart" diff --git a/compose/rest/data_privacy.go b/compose/rest/data_privacy.go index 4176f705b..bfe82a1a6 100644 --- a/compose/rest/data_privacy.go +++ b/compose/rest/data_privacy.go @@ -2,10 +2,10 @@ package rest import ( "context" - "github.com/cortezaproject/corteza-server/compose/rest/request" "github.com/cortezaproject/corteza-server/compose/service" "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/pkg/payload" ) @@ -28,6 +28,11 @@ type ( Values []map[string]any `json:"values"` } + privacyModuleSetPayload struct { + Filter types.PrivacyModuleFilter `json:"filter"` + Set []*types.PrivacyModule `json:"set"` + } + privateDataFinder interface { FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) } @@ -36,6 +41,7 @@ type ( record privateDataFinder module service.ModuleService namespace service.NamespaceService + privacy service.DataPrivacyService } ) @@ -44,6 +50,7 @@ func (DataPrivacy) New() *DataPrivacy { record: service.DefaultRecord, module: service.DefaultModule, namespace: service.DefaultNamespace, + privacy: service.DefaultDataPrivacy, } } @@ -113,3 +120,34 @@ func (ctrl *DataPrivacy) SensitiveDataList(ctx context.Context, r *request.DataP return outSet, nil } + +func (ctrl *DataPrivacy) ModuleList(ctx context.Context, r *request.DataPrivacyModuleList) (out interface{}, err error) { + var ( + f = types.PrivacyModuleFilter{ + ConnectionID: payload.ParseUint64s(r.ConnectionID), + } + ) + + if f.Paging, err = filter.NewPaging(r.Limit, r.PageCursor); err != nil { + return nil, err + } + + if f.Sorting, err = filter.NewSorting(r.Sort); err != nil { + return nil, err + } + + set, f, err := ctrl.privacy.FindModules(ctx, f) + return ctrl.makeFilterPayload(ctx, set, f, err) +} + +func (ctrl DataPrivacy) makeFilterPayload(_ context.Context, mm types.PrivacyModuleSet, f types.PrivacyModuleFilter, err error) (*privacyModuleSetPayload, error) { + if err != nil { + return nil, err + } + + if len(mm) == 0 { + mm = make([]*types.PrivacyModule, 0) + } + + return &privacyModuleSetPayload{Filter: f, Set: mm}, nil +} diff --git a/compose/rest/handlers/dataPrivacy.go b/compose/rest/handlers/dataPrivacy.go index 099494485..a1b6a2b57 100644 --- a/compose/rest/handlers/dataPrivacy.go +++ b/compose/rest/handlers/dataPrivacy.go @@ -20,11 +20,13 @@ type ( // Internal API interface DataPrivacyAPI interface { SensitiveDataList(context.Context, *request.DataPrivacySensitiveDataList) (interface{}, error) + ModuleList(context.Context, *request.DataPrivacyModuleList) (interface{}, error) } // HTTP API interface DataPrivacy struct { SensitiveDataList func(http.ResponseWriter, *http.Request) + ModuleList func(http.ResponseWriter, *http.Request) } ) @@ -44,6 +46,22 @@ func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy { return } + api.Send(w, r, value) + }, + ModuleList: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewDataPrivacyModuleList() + if err := params.Fill(r); err != nil { + api.Send(w, r, err) + return + } + + value, err := h.ModuleList(r.Context(), params) + if err != nil { + api.Send(w, r, err) + return + } + api.Send(w, r, value) }, } @@ -53,5 +71,6 @@ func (h DataPrivacy) MountRoutes(r chi.Router, middlewares ...func(http.Handler) r.Group(func(r chi.Router) { r.Use(middlewares...) r.Get("/data-privacy/sensitive-data", h.SensitiveDataList) + r.Get("/data-privacy/module", h.ModuleList) }) } diff --git a/compose/rest/request/dataPrivacy.go b/compose/rest/request/dataPrivacy.go index 28013b8ca..2cb958dac 100644 --- a/compose/rest/request/dataPrivacy.go +++ b/compose/rest/request/dataPrivacy.go @@ -44,6 +44,28 @@ type ( // Filter by connection ID ConnectionID []string } + + DataPrivacyModuleList struct { + // ConnectionID GET parameter + // + // Filter by connection ID + ConnectionID []string + + // Limit GET parameter + // + // Limit + Limit uint + + // PageCursor GET parameter + // + // Page cursor + PageCursor string + + // Sort GET parameter + // + // Sort items + Sort string + } ) // NewDataPrivacySensitiveDataList request @@ -97,3 +119,79 @@ func (r *DataPrivacySensitiveDataList) Fill(req *http.Request) (err error) { return err } + +// NewDataPrivacyModuleList request +func NewDataPrivacyModuleList() *DataPrivacyModuleList { + return &DataPrivacyModuleList{} +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyModuleList) Auditable() map[string]interface{} { + return map[string]interface{}{ + "connectionID": r.ConnectionID, + "limit": r.Limit, + "pageCursor": r.PageCursor, + "sort": r.Sort, + } +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyModuleList) GetConnectionID() []string { + return r.ConnectionID +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyModuleList) GetLimit() uint { + return r.Limit +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyModuleList) GetPageCursor() string { + return r.PageCursor +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyModuleList) GetSort() string { + return r.Sort +} + +// Fill processes request and fills internal variables +func (r *DataPrivacyModuleList) Fill(req *http.Request) (err error) { + + { + // GET params + tmp := req.URL.Query() + + if val, ok := tmp["connectionID[]"]; ok { + r.ConnectionID, err = val, nil + if err != nil { + return err + } + } else if val, ok := tmp["connectionID"]; ok { + r.ConnectionID, err = val, nil + if err != nil { + return err + } + } + if val, ok := tmp["limit"]; ok && len(val) > 0 { + r.Limit, err = payload.ParseUint(val[0]), nil + if err != nil { + return err + } + } + if val, ok := tmp["pageCursor"]; ok && len(val) > 0 { + r.PageCursor, err = val[0], nil + if err != nil { + return err + } + } + if val, ok := tmp["sort"]; ok && len(val) > 0 { + r.Sort, err = val[0], nil + if err != nil { + return err + } + } + } + + return err +} diff --git a/compose/service/data_privacy.go b/compose/service/data_privacy.go new file mode 100644 index 000000000..ab18e1711 --- /dev/null +++ b/compose/service/data_privacy.go @@ -0,0 +1,93 @@ +package service + +import ( + "context" + "github.com/cortezaproject/corteza-server/compose/types" + sysService "github.com/cortezaproject/corteza-server/system/service" + sysTypes "github.com/cortezaproject/corteza-server/system/types" +) + +type ( + dataPrivacy struct { + ns NamespaceService + m ModuleService + dalConn dalConnectionService + } + + dalConnectionService interface { + FindByID(ctx context.Context, ID uint64) (q *sysTypes.DalConnection, err error) + } + + moduleSetPayload struct { + Set []types.PrivacyModule `json:"set"` + } + + DataPrivacyService interface { + FindModules(ctx context.Context, filter types.PrivacyModuleFilter) (types.PrivacyModuleSet, types.PrivacyModuleFilter, error) + } +) + +func DataPrivacy() *dataPrivacy { + return &dataPrivacy{ + ns: DefaultNamespace, + m: DefaultModule, + dalConn: sysService.DefaultDalConnection, + } +} + +func (svc dataPrivacy) FindModules(ctx context.Context, filter types.PrivacyModuleFilter) (out types.PrivacyModuleSet, f types.PrivacyModuleFilter, err error) { + var ( + modules []types.PrivacyModule + cc = make(map[uint64]*sysTypes.DalConnection, 0) + ) + + reqConnes := make(map[uint64]bool) + hasReqConnes := len(filter.ConnectionID) > 0 + for _, connectionID := range filter.ConnectionID { + reqConnes[connectionID] = true + } + + // All namespaces + namespaces, _, err := svc.ns.Find(ctx, types.NamespaceFilter{}) + if err != nil { + return + } + + for _, n := range namespaces { + // Sensitive modules only + modules, f, err = svc.m.FindSensitive(ctx, types.PrivacyModuleFilter{NamespaceID: n.ID}) + if err != nil { + return + } + if len(modules) == 0 { + continue + } + + for _, m := range modules { + connID := m.ConnectionID + if hasReqConnes && !reqConnes[connID] { + continue + } + + var c *sysTypes.DalConnection + if val, ok := cc[connID]; ok { + c = val + } else { + c, err = svc.dalConn.FindByID(ctx, connID) + if err != nil { + cc[connID] = c + } + } + + out = append(out, &types.PrivacyModule{ + ID: m.ID, + Name: m.Name, + Handle: m.Handle, + Owner: m.Owner, + Connection: c, + }) + } + } + + return out, f, nil +} diff --git a/compose/service/module.go b/compose/service/module.go index 509550e4d..641892400 100644 --- a/compose/service/module.go +++ b/compose/service/module.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/dal" "reflect" "sort" @@ -49,6 +50,7 @@ type ( FindByHandle(ctx context.Context, namespaceID uint64, handle string) (*types.Module, error) FindByAny(ctx context.Context, namespaceID uint64, identifier interface{}) (*types.Module, error) Find(ctx context.Context, filter types.ModuleFilter) (set types.ModuleSet, f types.ModuleFilter, err error) + FindSensitive(ctx context.Context, filter types.PrivacyModuleFilter) (set []types.PrivacyModule, f types.PrivacyModuleFilter, err error) Create(ctx context.Context, module *types.Module) (*types.Module, error) Update(ctx context.Context, module *types.Module) (*types.Module, error) @@ -382,6 +384,46 @@ func (svc *module) ReloadDALModels(ctx context.Context) (err error) { return dalutils.ComposeModulesReload(ctx, svc.store, svc.dal) } +// FindSensitive will list all module with at least one private module field +func (svc module) FindSensitive(ctx context.Context, filter types.PrivacyModuleFilter) (set []types.PrivacyModule, f types.PrivacyModuleFilter, err error) { + var ( + mm types.ModuleSet + ) + + err = func() error { + mm, _, err = svc.Find(ctx, types.ModuleFilter{NamespaceID: filter.NamespaceID}) + if err != nil { + return err + } + + identity := auth.GetIdentityFromContext(ctx).Identity() + + for _, m := range mm { + isPrivate := false + for _, f := range m.Fields { + if !isPrivate { + isPrivate = f.Private + } + } + + if isPrivate && m != nil { + ownedBy, _ := m.ModelConfig.SystemFieldEncoding.OwnedBy.Value() + set = append(set, types.PrivacyModule{ + ID: m.ID, + Name: m.Name, // @todo get this as per translation + Handle: m.Handle, + Owner: ownedBy == identity, + ConnectionID: m.ModelConfig.ConnectionID, + }) + } + } + + return nil + }() + + return set, filter, err +} + func (svc module) updater(ctx context.Context, namespaceID, moduleID uint64, action func(...*moduleActionProps) *moduleAction, fn moduleUpdateHandler) (*types.Module, error) { var ( changes moduleChanges @@ -857,7 +899,7 @@ func loadModuleFields(ctx context.Context, s store.Storer, mm ...*types.Module) for _, m := range mm { m.Fields = ff.FilterByModule(m.ID) - m.Fields.Walk(func(f *types.ModuleField) error { + _ = m.Fields.Walk(func(f *types.ModuleField) error { f.NamespaceID = m.NamespaceID return nil }) diff --git a/compose/service/service.go b/compose/service/service.go index eff48ac59..169f0d08e 100644 --- a/compose/service/service.go +++ b/compose/service/service.go @@ -71,6 +71,7 @@ var ( DefaultAttachment AttachmentService DefaultNotification *notification DefaultResourceTranslation ResourceTranslationsManagerService + DefaultDataPrivacy DataPrivacyService // wrapper around time.Now() that will aid service testing now = func() *time.Time { @@ -196,6 +197,7 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, c Config) DefaultChart = Chart() DefaultNotification = Notification(c.UserFinder) DefaultAttachment = Attachment(DefaultObjectStore, dal.Service()) + DefaultDataPrivacy = DataPrivacy() RegisterIteratorProviders() diff --git a/compose/types/data_privacy.go b/compose/types/data_privacy.go new file mode 100644 index 000000000..a8b815401 --- /dev/null +++ b/compose/types/data_privacy.go @@ -0,0 +1,27 @@ +package types + +import ( + "github.com/cortezaproject/corteza-server/pkg/filter" + sysTypes "github.com/cortezaproject/corteza-server/system/types" +) + +type ( + PrivacyModule struct { + ID uint64 `json:"moduleID,string"` + Name string `json:"name"` + Handle string `json:"handle"` + Owner bool `json:"owner"` + + ConnectionID uint64 `json:"-"` + Connection *sysTypes.DalConnection + } + + PrivacyModuleFilter struct { + NamespaceID uint64 `json:"namespaceID,string"` + ConnectionID []uint64 `json:"connectionID,string"` + + // Standard helpers for paging and sorting + filter.Sorting + filter.Paging + } +) diff --git a/compose/types/type_set.gen.go b/compose/types/type_set.gen.go index 36b996f31..d29ff9490 100644 --- a/compose/types/type_set.gen.go +++ b/compose/types/type_set.gen.go @@ -40,6 +40,11 @@ type ( // This type is auto-generated. PageSet []*Page + // PrivacyModuleSet slice of PrivacyModule + // + // This type is auto-generated. + PrivacyModuleSet []*PrivacyModule + // RecordSet slice of Record // // This type is auto-generated. @@ -387,6 +392,36 @@ func (set PageSet) IDs() (IDs []uint64) { return } +// Walk iterates through every slice item and calls w(PrivacyModule) err +// +// This function is auto-generated. +func (set PrivacyModuleSet) Walk(w func(*PrivacyModule) error) (err error) { + for i := range set { + if err = w(set[i]); err != nil { + return + } + } + + return +} + +// Filter iterates through every slice item, calls f(PrivacyModule) (bool, err) and return filtered slice +// +// This function is auto-generated. +func (set PrivacyModuleSet) Filter(f func(*PrivacyModule) (bool, error)) (out PrivacyModuleSet, err error) { + var ok bool + out = PrivacyModuleSet{} + for i := range set { + if ok, err = f(set[i]); err != nil { + return + } else if ok { + out = append(out, set[i]) + } + } + + return +} + // Walk iterates through every slice item and calls w(Record) err // // This function is auto-generated. diff --git a/compose/types/type_set.gen_test.go b/compose/types/type_set.gen_test.go index 022ed4aa3..6b38e4918 100644 --- a/compose/types/type_set.gen_test.go +++ b/compose/types/type_set.gen_test.go @@ -554,6 +554,62 @@ func TestPageSetIDs(t *testing.T) { } } +func TestPrivacyModuleSetWalk(t *testing.T) { + var ( + value = make(PrivacyModuleSet, 3) + req = require.New(t) + ) + + // check walk with no errors + { + err := value.Walk(func(*PrivacyModule) error { + return nil + }) + req.NoError(err) + } + + // check walk with error + req.Error(value.Walk(func(*PrivacyModule) error { return fmt.Errorf("walk error") })) +} + +func TestPrivacyModuleSetFilter(t *testing.T) { + var ( + value = make(PrivacyModuleSet, 3) + req = require.New(t) + ) + + // filter nothing + { + set, err := value.Filter(func(*PrivacyModule) (bool, error) { + return true, nil + }) + req.NoError(err) + req.Equal(len(set), len(value)) + } + + // filter one item + { + found := false + set, err := value.Filter(func(*PrivacyModule) (bool, error) { + if !found { + found = true + return found, nil + } + return false, nil + }) + req.NoError(err) + req.Len(set, 1) + } + + // filter error + { + _, err := value.Filter(func(*PrivacyModule) (bool, error) { + return false, fmt.Errorf("filter error") + }) + req.Error(err) + } +} + func TestRecordSetWalk(t *testing.T) { var ( value = make(RecordSet, 3) diff --git a/compose/types/types.yaml b/compose/types/types.yaml index 9e1e2db61..765d0b04b 100644 --- a/compose/types/types.yaml +++ b/compose/types/types.yaml @@ -14,4 +14,6 @@ types: labelResourceType: compose:record RecordValue: noIdField: true + PrivacyModule: + noIdField: true diff --git a/pkg/dal/service.go b/pkg/dal/service.go index 5c6bd423c..08cb6b297 100644 --- a/pkg/dal/service.go +++ b/pkg/dal/service.go @@ -313,7 +313,7 @@ func (svc *service) ReplaceConnection(ctx context.Context, cw *ConnectionWrap, i return nil } -// DeleteConnection removes the given connection from the DAL +// RemoveConnection removes the given connection from the DAL func (svc *service) RemoveConnection(ctx context.Context, ID uint64) (err error) { var ( issues = newIssueHelper().addConnection(ID)