diff --git a/compose/rest.yaml b/compose/rest.yaml index 6b7dd0029..b47dcf1a0 100644 --- a/compose/rest.yaml +++ b/compose/rest.yaml @@ -978,10 +978,10 @@ endpoints: entrypoint: dataPrivacy path: "/data-privacy" apis: - - name: sensitive data list + - name: record list method: GET - title: List sensitive data - path: /sensitive-data + title: List records for data privacy + path: /record parameters: get: - { name: sensitivityLevelID, type: "uint64", title: "Sensitivity Level ID", required: false} diff --git a/compose/rest/data_privacy.go b/compose/rest/data_privacy.go index bfe82a1a6..ed15303f6 100644 --- a/compose/rest/data_privacy.go +++ b/compose/rest/data_privacy.go @@ -2,6 +2,7 @@ package rest import ( "context" + "github.com/cortezaproject/corteza-server/compose/rest/request" "github.com/cortezaproject/corteza-server/compose/service" "github.com/cortezaproject/corteza-server/compose/types" @@ -10,7 +11,7 @@ import ( ) type ( - sensitiveDataSetPayload struct { + sensitiveRecordsSetPayload struct { Set []*sensitiveDataPayload `json:"set"` } @@ -20,10 +21,10 @@ type ( ModuleID uint64 `json:"moduleID,string"` Module string `json:"module"` - Records []sensitiveData `json:"records"` + Records []sensitiveRecords `json:"records"` } - sensitiveData struct { + sensitiveRecords struct { RecordID uint64 `json:"recordID,string"` Values []map[string]any `json:"values"` } @@ -34,7 +35,7 @@ type ( } privateDataFinder interface { - FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) + FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error) } DataPrivacy struct { @@ -54,68 +55,51 @@ func (DataPrivacy) New() *DataPrivacy { } } -func (ctrl *DataPrivacy) SensitiveDataList(ctx context.Context, r *request.DataPrivacySensitiveDataList) (out interface{}, err error) { - outSet := sensitiveDataSetPayload{} - +func (ctrl *DataPrivacy) RecordList(ctx context.Context, r *request.DataPrivacyRecordList) (out interface{}, err error) { + // If we're requesting only specific connections, prepare filter params here reqConns := make(map[uint64]bool) hasReqConns := len(r.ConnectionID) > 0 for _, connectionID := range payload.ParseUint64s(r.ConnectionID) { reqConns[connectionID] = true } - // All namespaces - namespaces, _, err := ctrl.namespace.Find(ctx, types.NamespaceFilter{}) + // Collect sensitive records + ss, err := ctrl.record.FindSensitive(ctx) if err != nil { return } - outSet.Set = make([]*sensitiveDataPayload, 0, 10) + outSet := sensitiveRecordsSetPayload{ + Set: make([]*sensitiveDataPayload, 0, 10), + } - for _, n := range namespaces { - // All modules - modules, _, err := ctrl.module.Find(ctx, types.ModuleFilter{NamespaceID: n.ID}) - if err != nil { - return nil, err + for _, s := range ss { + // Skip the ones we don't want + if hasReqConns && !reqConns[s.Module.ModelConfig.ConnectionID] { + continue } - for _, m := range modules { - conn := m.ModelConfig.ConnectionID - if hasReqConns && !reqConns[conn] { - continue - } - sData, err := ctrl.record.FindSensitive(ctx, types.RecordFilter{ModuleID: m.ID, NamespaceID: m.NamespaceID}) - if err != nil { - return nil, err - } - if len(sData) == 0 { - continue - } + // Build the payload + payload := &sensitiveDataPayload{ + NamespaceID: s.Namespace.ID, + Namespace: s.Namespace.Name, - nsMod := &sensitiveDataPayload{ - NamespaceID: n.ID, - Namespace: n.Name, + ModuleID: s.Module.ID, + Module: s.Module.Name, - ModuleID: m.ID, - Module: m.Name, - - Records: make([]sensitiveData, 0, len(sData)), - } - for _, a := range sData { - if len(a.Values) == 0 { - continue - } - nsMod.Records = append(nsMod.Records, sensitiveData{ - RecordID: a.ID, - Values: a.Values, - }) - } - - if len(nsMod.Records) == 0 { - continue - } - - outSet.Set = append(outSet.Set, nsMod) + Records: make([]sensitiveRecords, 0, len(s.Records)), } + for _, a := range s.Records { + if len(a.Values) == 0 { + continue + } + payload.Records = append(payload.Records, sensitiveRecords{ + RecordID: a.RecordID, + Values: a.Values, + }) + } + + outSet.Set = append(outSet.Set, payload) } return outSet, nil diff --git a/compose/rest/handlers/dataPrivacy.go b/compose/rest/handlers/dataPrivacy.go index a1b6a2b57..4f3b2767c 100644 --- a/compose/rest/handlers/dataPrivacy.go +++ b/compose/rest/handlers/dataPrivacy.go @@ -19,28 +19,28 @@ import ( type ( // Internal API interface DataPrivacyAPI interface { - SensitiveDataList(context.Context, *request.DataPrivacySensitiveDataList) (interface{}, error) + RecordList(context.Context, *request.DataPrivacyRecordList) (interface{}, error) ModuleList(context.Context, *request.DataPrivacyModuleList) (interface{}, error) } // HTTP API interface DataPrivacy struct { - SensitiveDataList func(http.ResponseWriter, *http.Request) - ModuleList func(http.ResponseWriter, *http.Request) + RecordList func(http.ResponseWriter, *http.Request) + ModuleList func(http.ResponseWriter, *http.Request) } ) func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy { return &DataPrivacy{ - SensitiveDataList: func(w http.ResponseWriter, r *http.Request) { + RecordList: func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - params := request.NewDataPrivacySensitiveDataList() + params := request.NewDataPrivacyRecordList() if err := params.Fill(r); err != nil { api.Send(w, r, err) return } - value, err := h.SensitiveDataList(r.Context(), params) + value, err := h.RecordList(r.Context(), params) if err != nil { api.Send(w, r, err) return @@ -70,7 +70,7 @@ func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy { func (h DataPrivacy) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) { r.Group(func(r chi.Router) { r.Use(middlewares...) - r.Get("/data-privacy/sensitive-data", h.SensitiveDataList) + r.Get("/data-privacy/record", h.RecordList) r.Get("/data-privacy/module", h.ModuleList) }) } diff --git a/compose/rest/request/dataPrivacy.go b/compose/rest/request/dataPrivacy.go index 2cb958dac..b313e9eea 100644 --- a/compose/rest/request/dataPrivacy.go +++ b/compose/rest/request/dataPrivacy.go @@ -33,7 +33,7 @@ var ( type ( // Internal API interface - DataPrivacySensitiveDataList struct { + DataPrivacyRecordList struct { // SensitivityLevelID GET parameter // // Sensitivity Level ID @@ -68,13 +68,13 @@ type ( } ) -// NewDataPrivacySensitiveDataList request -func NewDataPrivacySensitiveDataList() *DataPrivacySensitiveDataList { - return &DataPrivacySensitiveDataList{} +// NewDataPrivacyRecordList request +func NewDataPrivacyRecordList() *DataPrivacyRecordList { + return &DataPrivacyRecordList{} } // Auditable returns all auditable/loggable parameters -func (r DataPrivacySensitiveDataList) Auditable() map[string]interface{} { +func (r DataPrivacyRecordList) Auditable() map[string]interface{} { return map[string]interface{}{ "sensitivityLevelID": r.SensitivityLevelID, "connectionID": r.ConnectionID, @@ -82,17 +82,17 @@ func (r DataPrivacySensitiveDataList) Auditable() map[string]interface{} { } // Auditable returns all auditable/loggable parameters -func (r DataPrivacySensitiveDataList) GetSensitivityLevelID() uint64 { +func (r DataPrivacyRecordList) GetSensitivityLevelID() uint64 { return r.SensitivityLevelID } // Auditable returns all auditable/loggable parameters -func (r DataPrivacySensitiveDataList) GetConnectionID() []string { +func (r DataPrivacyRecordList) GetConnectionID() []string { return r.ConnectionID } // Fill processes request and fills internal variables -func (r *DataPrivacySensitiveDataList) Fill(req *http.Request) (err error) { +func (r *DataPrivacyRecordList) Fill(req *http.Request) (err error) { { // GET params diff --git a/compose/service/module.go b/compose/service/module.go index 970c952e0..b4a782c5d 100644 --- a/compose/service/module.go +++ b/compose/service/module.go @@ -444,9 +444,7 @@ func (svc module) FindSensitive(ctx context.Context, filter types.PrivacyModuleF for _, m := range mm { isPrivate := false for _, f := range m.Fields { - if !isPrivate { - isPrivate = f.Private - } + isPrivate = isPrivate || f.IsSensitive() } if isPrivate && m != nil { diff --git a/compose/service/record.go b/compose/service/record.go index 0be2dd159..47c83339c 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -44,6 +44,9 @@ type ( store store.Storer + namespace namespaceFinder + module moduleFinder + formatter recordValuesFormatter sanitizer recordValuesSanitizer validator recordValuesValidator @@ -88,12 +91,20 @@ type ( recordValueAccessController } + moduleFinder interface { + Find(ctx context.Context, filter types.ModuleFilter) (set types.ModuleSet, f types.ModuleFilter, err error) + } + + namespaceFinder interface { + Find(context.Context, types.NamespaceFilter) (types.NamespaceSet, types.NamespaceFilter, error) + } + RecordService interface { FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, error) Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error) Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) - FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) + FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error) RecordExport(context.Context, types.RecordFilter) error RecordImport(context.Context, error) error @@ -163,6 +174,9 @@ func Record() *record { store: DefaultStore, dal: dal.Service(), + namespace: DefaultNamespace, + module: DefaultModule, + formatter: values.Formatter(), sanitizer: values.Sanitizer(), } @@ -327,53 +341,44 @@ func (svc record) Find(ctx context.Context, filter types.RecordFilter) (set type return set, f, svc.recordAction(ctx, aProps, RecordActionSearch, err) } -func (svc record) FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) { +// FindSensitive returns stripped down records for all namespaces/modules where fields define a sensitivity level +func (svc record) FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error) { var ( - m *types.Module + namespaces types.NamespaceSet + modules types.ModuleSet + + userID = auth.GetIdentityFromContext(ctx).Identity() ) err = func() error { - if m, err = loadModule(ctx, svc.store, filter.NamespaceID, filter.ModuleID); err != nil { - return err - } - - // Force the query to only show owned records - // @todo allow additional querying - filter.Query = fmt.Sprintf("ownedBy='%d'", auth.GetIdentityFromContext(ctx).Identity()) - - rr, _, err := svc.Find(ctx, filter) + // Get namespaces + namespaces, _, err = svc.namespace.Find(ctx, types.NamespaceFilter{}) if err != nil { return err } - for _, r := range rr { - vv := make([]map[string]any, 0, len(r.Values)) + for _, namespace := range namespaces { + // Get corresponding modules + modules, _, err = svc.module.Find(ctx, types.ModuleFilter{NamespaceID: namespace.ID}) + if err != nil { + return err + } - for _, f := range m.Fields { - // Skip the ones with no privacy - // @todo allow the request to specify what level we wish to see - if !f.Private { + for _, module := range modules { + // Get sensitive record data + aux, err := svc.findSensitive(ctx, userID, namespace, module, types.RecordFilter{ + ModuleID: module.ID, + NamespaceID: namespace.ID, + }) + if err != nil { + return err + } + if len(aux.Records) == 0 { continue } - values := make([]any, 0, 2) - for _, v := range r.Values.FilterByName(f.Name) { - values = append(values, v.Value) - } - - // Make value - vv = append(vv, map[string]any{ - "name": f.Name, - "kind": f.Kind, - "isMulti": f.Multi, - "value": values, - }) + set = append(set, aux) } - - set = append(set, types.PrivateDataSet{ - ID: r.ID, - Values: vv, - }) } return nil @@ -382,6 +387,54 @@ func (svc record) FindSensitive(ctx context.Context, filter types.RecordFilter) return set, err } +func (svc record) findSensitive(ctx context.Context, userID uint64, namespace *types.Namespace, module *types.Module, filter types.RecordFilter) (out types.SensitiveRecordSet, err error) { + out = types.SensitiveRecordSet{ + Namespace: namespace, + Module: module, + } + + // Force the query to only show owned records + // @todo allow additional querying + filter.Query = fmt.Sprintf("ownedBy='%d'", userID) + + rr, _, err := svc.Find(ctx, filter) + if err != nil { + return + } + + for _, r := range rr { + vv := make([]map[string]any, 0, len(r.Values)) + + for _, f := range module.Fields { + // Skip the ones with no privacy + // @todo allow the request to specify what level we wish to see + if !f.IsSensitive() { + continue + } + + values := make([]any, 0, 2) + for _, v := range r.Values.FilterByName(f.Name) { + values = append(values, v.Value) + } + + // Make value + vv = append(vv, map[string]any{ + "name": f.Name, + "kind": f.Kind, + "isMulti": f.Multi, + "value": values, + }) + } + + out.Records = append(out.Records, types.SensitiveRecord{ + RecordID: r.ID, + Values: vv, + }) + } + + return +} + func (svc record) RecordImport(ctx context.Context, err error) error { return svc.recordAction(ctx, &recordActionProps{}, RecordActionImport, err) } diff --git a/compose/types/module_field.go b/compose/types/module_field.go index dba389ad4..68a78efc5 100644 --- a/compose/types/module_field.go +++ b/compose/types/module_field.go @@ -516,3 +516,7 @@ func (f ModuleField) IsTimeOnly() bool { func (f ModuleField) IsRef() bool { return f.Kind == "Record" || f.Kind == "User" || f.Kind == "File" } + +func (f ModuleField) IsSensitive() bool { + return f.Privacy.SensitivityLevel > 0 +} diff --git a/compose/types/record.go b/compose/types/record.go index 5867b83f3..2589693b8 100644 --- a/compose/types/record.go +++ b/compose/types/record.go @@ -80,9 +80,18 @@ type ( RecordFilter } - PrivateDataSet struct { - ID uint64 - Values []map[string]any + SensitiveRecord struct { + RecordID uint64 + Values []map[string]any + } + + SensitiveRecordSet struct { + // Contextual metadata + ConnectionID uint64 + Module *Module + Namespace *Namespace + + Records []SensitiveRecord } )