DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Add route for listing sensitive modules

Browse Source 
It will list out all the privacy related modules, that have one or more private fields.
This commit is contained in:
Vivek Patel
2022-07-18 17:24:37 +05:30
parent 07a8fdbbd5
commit 56a7bfa821
12 changed files with 425 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/rest.yaml
+10
View File
@@ -986,6 +986,16 @@ endpoints:
get:
- { name: sensitivityLevelID, type: "uint64", title: "Sensitivity Level ID", required: false}
- { name: connectionID, type: "[]string", required: false, title: "Filter by connection ID"}
- name: module list
method: GET
title: List modules
path: /module
parameters:
get:
- { name: connectionID, type: "[]string", title: "Filter by connection ID" }
- { name: limit, type: "uint", title: "Limit" }
- { name: pageCursor, type: "string", title: "Page cursor" }
- { name: sort, type: "string", title: "Sort items" }
- title: Charts
path: "/namespace/{namespaceID}/chart"
compose/rest/data_privacy.go
+39 -1
View File
@@ -2,10 +2,10 @@ package rest
import (
"context"
"github.com/cortezaproject/corteza-server/compose/rest/request"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/pkg/payload"
)
@@ -28,6 +28,11 @@ type (
Values []map[string]any `json:"values"`
}
privacyModuleSetPayload struct {
Filter types.PrivacyModuleFilter `json:"filter"`
Set []*types.PrivacyModule `json:"set"`
}
privateDataFinder interface {
FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error)
}
@@ -36,6 +41,7 @@ type (
record privateDataFinder
module service.ModuleService
namespace service.NamespaceService
privacy service.DataPrivacyService
}
)
@@ -44,6 +50,7 @@ func (DataPrivacy) New() *DataPrivacy {
record: service.DefaultRecord,
module: service.DefaultModule,
namespace: service.DefaultNamespace,
privacy: service.DefaultDataPrivacy,
}
}
@@ -113,3 +120,34 @@ func (ctrl *DataPrivacy) SensitiveDataList(ctx context.Context, r *request.DataP
return outSet, nil
}
func (ctrl *DataPrivacy) ModuleList(ctx context.Context, r *request.DataPrivacyModuleList) (out interface{}, err error) {
var (
f = types.PrivacyModuleFilter{
ConnectionID: payload.ParseUint64s(r.ConnectionID),
}
)
if f.Paging, err = filter.NewPaging(r.Limit, r.PageCursor); err != nil {
return nil, err
}
if f.Sorting, err = filter.NewSorting(r.Sort); err != nil {
return nil, err
}
set, f, err := ctrl.privacy.FindModules(ctx, f)
return ctrl.makeFilterPayload(ctx, set, f, err)
}
func (ctrl DataPrivacy) makeFilterPayload(_ context.Context, mm types.PrivacyModuleSet, f types.PrivacyModuleFilter, err error) (*privacyModuleSetPayload, error) {
if err != nil {
return nil, err
}
if len(mm) == 0 {
mm = make([]*types.PrivacyModule, 0)
}
return &privacyModuleSetPayload{Filter: f, Set: mm}, nil
}
compose/rest/handlers/dataPrivacy.go
+19
View File
@@ -20,11 +20,13 @@ type (
// Internal API interface
DataPrivacyAPI interface {
SensitiveDataList(context.Context, *request.DataPrivacySensitiveDataList) (interface{}, error)
ModuleList(context.Context, *request.DataPrivacyModuleList) (interface{}, error)
}
// HTTP API interface
DataPrivacy struct {
SensitiveDataList func(http.ResponseWriter, *http.Request)
ModuleList func(http.ResponseWriter, *http.Request)
}
)
@@ -44,6 +46,22 @@ func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy {
return
}
api.Send(w, r, value)
},
ModuleList: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewDataPrivacyModuleList()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.ModuleList(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
}
api.Send(w, r, value)
},
}
@@ -53,5 +71,6 @@ func (h DataPrivacy) MountRoutes(r chi.Router, middlewares ...func(http.Handler)
r.Group(func(r chi.Router) {
r.Use(middlewares...)
r.Get("/data-privacy/sensitive-data", h.SensitiveDataList)
r.Get("/data-privacy/module", h.ModuleList)
})
}
compose/rest/request/dataPrivacy.go
+98
View File
@@ -44,6 +44,28 @@ type (
// Filter by connection ID
ConnectionID []string
}
DataPrivacyModuleList struct {
// ConnectionID GET parameter
//
// Filter by connection ID
ConnectionID []string
// Limit GET parameter
//
// Limit
Limit uint
// PageCursor GET parameter
//
// Page cursor
PageCursor string
// Sort GET parameter
//
// Sort items
Sort string
}
)
// NewDataPrivacySensitiveDataList request
@@ -97,3 +119,79 @@ func (r *DataPrivacySensitiveDataList) Fill(req *http.Request) (err error) {
return err
}
// NewDataPrivacyModuleList request
func NewDataPrivacyModuleList() *DataPrivacyModuleList {
return &DataPrivacyModuleList{}
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacyModuleList) Auditable() map[string]interface{} {
return map[string]interface{}{
"connectionID": r.ConnectionID,
"limit": r.Limit,
"pageCursor": r.PageCursor,
"sort": r.Sort,
}
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacyModuleList) GetConnectionID() []string {
return r.ConnectionID
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacyModuleList) GetLimit() uint {
return r.Limit
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacyModuleList) GetPageCursor() string {
return r.PageCursor
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacyModuleList) GetSort() string {
return r.Sort
}
// Fill processes request and fills internal variables
func (r *DataPrivacyModuleList) Fill(req *http.Request) (err error) {
{
// GET params
tmp := req.URL.Query()
if val, ok := tmp["connectionID[]"]; ok {
r.ConnectionID, err = val, nil
if err != nil {
return err
}
} else if val, ok := tmp["connectionID"]; ok {
r.ConnectionID, err = val, nil
if err != nil {
return err
}
}
if val, ok := tmp["limit"]; ok && len(val) > 0 {
r.Limit, err = payload.ParseUint(val[0]), nil
if err != nil {
return err
}
}
if val, ok := tmp["pageCursor"]; ok && len(val) > 0 {
r.PageCursor, err = val[0], nil
if err != nil {
return err
}
}
if val, ok := tmp["sort"]; ok && len(val) > 0 {
r.Sort, err = val[0], nil
if err != nil {
return err
}
}
}
return err
}
compose/service/data_privacy.go
+93
View File
@@ -0,0 +1,93 @@
package service
import (
"context"
"github.com/cortezaproject/corteza-server/compose/types"
sysService "github.com/cortezaproject/corteza-server/system/service"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
type (
dataPrivacy struct {
ns NamespaceService
m ModuleService
dalConn dalConnectionService
}
dalConnectionService interface {
FindByID(ctx context.Context, ID uint64) (q *sysTypes.DalConnection, err error)
}
moduleSetPayload struct {
Set []types.PrivacyModule `json:"set"`
}
DataPrivacyService interface {
FindModules(ctx context.Context, filter types.PrivacyModuleFilter) (types.PrivacyModuleSet, types.PrivacyModuleFilter, error)
}
)
func DataPrivacy() *dataPrivacy {
return &dataPrivacy{
ns: DefaultNamespace,
m: DefaultModule,
dalConn: sysService.DefaultDalConnection,
}
}
func (svc dataPrivacy) FindModules(ctx context.Context, filter types.PrivacyModuleFilter) (out types.PrivacyModuleSet, f types.PrivacyModuleFilter, err error) {
var (
modules []types.PrivacyModule
cc = make(map[uint64]*sysTypes.DalConnection, 0)
)
reqConnes := make(map[uint64]bool)
hasReqConnes := len(filter.ConnectionID) > 0
for _, connectionID := range filter.ConnectionID {
reqConnes[connectionID] = true
}
// All namespaces
namespaces, _, err := svc.ns.Find(ctx, types.NamespaceFilter{})
if err != nil {
return
}
for _, n := range namespaces {
// Sensitive modules only
modules, f, err = svc.m.FindSensitive(ctx, types.PrivacyModuleFilter{NamespaceID: n.ID})
if err != nil {
return
}
if len(modules) == 0 {
continue
}
for _, m := range modules {
connID := m.ConnectionID
if hasReqConnes && !reqConnes[connID] {
continue
}
var c *sysTypes.DalConnection
if val, ok := cc[connID]; ok {
c = val
} else {
c, err = svc.dalConn.FindByID(ctx, connID)
if err != nil {
cc[connID] = c
}
}
out = append(out, &types.PrivacyModule{
ID: m.ID,
Name: m.Name,
Handle: m.Handle,
Owner: m.Owner,
Connection: c,
})
}
}
return out, f, nil
}
compose/service/module.go
+43 -1
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/dal"
"reflect"
"sort"
@@ -49,6 +50,7 @@ type (
FindByHandle(ctx context.Context, namespaceID uint64, handle string) (*types.Module, error)
FindByAny(ctx context.Context, namespaceID uint64, identifier interface{}) (*types.Module, error)
Find(ctx context.Context, filter types.ModuleFilter) (set types.ModuleSet, f types.ModuleFilter, err error)
FindSensitive(ctx context.Context, filter types.PrivacyModuleFilter) (set []types.PrivacyModule, f types.PrivacyModuleFilter, err error)
Create(ctx context.Context, module *types.Module) (*types.Module, error)
Update(ctx context.Context, module *types.Module) (*types.Module, error)
@@ -382,6 +384,46 @@ func (svc *module) ReloadDALModels(ctx context.Context) (err error) {
return dalutils.ComposeModulesReload(ctx, svc.store, svc.dal)
}
// FindSensitive will list all module with at least one private module field
func (svc module) FindSensitive(ctx context.Context, filter types.PrivacyModuleFilter) (set []types.PrivacyModule, f types.PrivacyModuleFilter, err error) {
var (
mm types.ModuleSet
)
err = func() error {
mm, _, err = svc.Find(ctx, types.ModuleFilter{NamespaceID: filter.NamespaceID})
if err != nil {
return err
}
identity := auth.GetIdentityFromContext(ctx).Identity()
for _, m := range mm {
isPrivate := false
for _, f := range m.Fields {
if !isPrivate {
isPrivate = f.Private
}
}
if isPrivate && m != nil {
ownedBy, _ := m.ModelConfig.SystemFieldEncoding.OwnedBy.Value()
set = append(set, types.PrivacyModule{
ID: m.ID,
Name: m.Name, // @todo get this as per translation
Handle: m.Handle,
Owner: ownedBy == identity,
ConnectionID: m.ModelConfig.ConnectionID,
})
}
}
return nil
}()
return set, filter, err
}
func (svc module) updater(ctx context.Context, namespaceID, moduleID uint64, action func(...*moduleActionProps) *moduleAction, fn moduleUpdateHandler) (*types.Module, error) {
var (
changes moduleChanges
@@ -857,7 +899,7 @@ func loadModuleFields(ctx context.Context, s store.Storer, mm ...*types.Module)
for _, m := range mm {
m.Fields = ff.FilterByModule(m.ID)
m.Fields.Walk(func(f *types.ModuleField) error {
_ = m.Fields.Walk(func(f *types.ModuleField) error {
f.NamespaceID = m.NamespaceID
return nil
})
compose/service/service.go
+2
View File
@@ -71,6 +71,7 @@ var (
DefaultAttachment AttachmentService
DefaultNotification *notification
DefaultResourceTranslation ResourceTranslationsManagerService
DefaultDataPrivacy DataPrivacyService
// wrapper around time.Now() that will aid service testing
now = func() *time.Time {
@@ -196,6 +197,7 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, c Config)
DefaultChart = Chart()
DefaultNotification = Notification(c.UserFinder)
DefaultAttachment = Attachment(DefaultObjectStore, dal.Service())
DefaultDataPrivacy = DataPrivacy()
RegisterIteratorProviders()
compose/types/data_privacy.go
+27
View File
@@ -0,0 +1,27 @@
package types
import (
"github.com/cortezaproject/corteza-server/pkg/filter"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
type (
PrivacyModule struct {
ID uint64 `json:"moduleID,string"`
Name string `json:"name"`
Handle string `json:"handle"`
Owner bool `json:"owner"`
ConnectionID uint64 `json:"-"`
Connection *sysTypes.DalConnection
}
PrivacyModuleFilter struct {
NamespaceID uint64 `json:"namespaceID,string"`
ConnectionID []uint64 `json:"connectionID,string"`
// Standard helpers for paging and sorting
filter.Sorting
filter.Paging
}
)
compose/types/type_set.gen.go
Generated
+35
View File
@@ -40,6 +40,11 @@ type (
// This type is auto-generated.
PageSet []*Page
// PrivacyModuleSet slice of PrivacyModule
//
// This type is auto-generated.
PrivacyModuleSet []*PrivacyModule
// RecordSet slice of Record
//
// This type is auto-generated.
@@ -387,6 +392,36 @@ func (set PageSet) IDs() (IDs []uint64) {
return
}
// Walk iterates through every slice item and calls w(PrivacyModule) err
//
// This function is auto-generated.
func (set PrivacyModuleSet) Walk(w func(*PrivacyModule) error) (err error) {
for i := range set {
if err = w(set[i]); err != nil {
return
}
}
return
}
// Filter iterates through every slice item, calls f(PrivacyModule) (bool, err) and return filtered slice
//
// This function is auto-generated.
func (set PrivacyModuleSet) Filter(f func(*PrivacyModule) (bool, error)) (out PrivacyModuleSet, err error) {
var ok bool
out = PrivacyModuleSet{}
for i := range set {
if ok, err = f(set[i]); err != nil {
return
} else if ok {
out = append(out, set[i])
}
}
return
}
// Walk iterates through every slice item and calls w(Record) err
//
// This function is auto-generated.
compose/types/type_set.gen_test.go
Generated
+56
View File
@@ -554,6 +554,62 @@ func TestPageSetIDs(t *testing.T) {
}
}
func TestPrivacyModuleSetWalk(t *testing.T) {
var (
value = make(PrivacyModuleSet, 3)
req = require.New(t)
)
// check walk with no errors
{
err := value.Walk(func(*PrivacyModule) error {
return nil
})
req.NoError(err)
}
// check walk with error
req.Error(value.Walk(func(*PrivacyModule) error { return fmt.Errorf("walk error") }))
}
func TestPrivacyModuleSetFilter(t *testing.T) {
var (
value = make(PrivacyModuleSet, 3)
req = require.New(t)
)
// filter nothing
{
set, err := value.Filter(func(*PrivacyModule) (bool, error) {
return true, nil
})
req.NoError(err)
req.Equal(len(set), len(value))
}
// filter one item
{
found := false
set, err := value.Filter(func(*PrivacyModule) (bool, error) {
if !found {
found = true
return found, nil
}
return false, nil
})
req.NoError(err)
req.Len(set, 1)
}
// filter error
{
_, err := value.Filter(func(*PrivacyModule) (bool, error) {
return false, fmt.Errorf("filter error")
})
req.Error(err)
}
}
func TestRecordSetWalk(t *testing.T) {
var (
value = make(RecordSet, 3)
compose/types/types.yaml
+2
View File
@@ -14,4 +14,6 @@ types:
labelResourceType: compose:record
RecordValue:
noIdField: true
PrivacyModule:
noIdField: true
pkg/dal/service.go
+1 -1
View File
@@ -313,7 +313,7 @@ func (svc *service) ReplaceConnection(ctx context.Context, cw *ConnectionWrap, i
return nil
}
// DeleteConnection removes the given connection from the DAL
// RemoveConnection removes the given connection from the DAL
func (svc *service) RemoveConnection(ctx context.Context, ID uint64) (err error) {
var (
issues = newIssueHelper().addConnection(ID)