DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Refactor data-privacy/sensitive-data to data-privacy/records

Browse Source
This commit is contained in:
Tomaž Jerman
2022-07-25 13:34:11 +02:00
parent 5e8fae37af
commit eb917eed6e
8 changed files with 157 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/rest.yaml
+3 -3
View File
@@ -978,10 +978,10 @@ endpoints:
entrypoint: dataPrivacy
path: "/data-privacy"
apis:
- name: sensitive data list
- name: record list
method: GET
title: List sensitive data
path: /sensitive-data
title: List records for data privacy
path: /record
parameters:
get:
- { name: sensitivityLevelID, type: "uint64", title: "Sensitivity Level ID", required: false}
compose/rest/data_privacy.go
+34 -50
View File
@@ -2,6 +2,7 @@ package rest
import (
"context"
"github.com/cortezaproject/corteza-server/compose/rest/request"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
@@ -10,7 +11,7 @@ import (
)
type (
sensitiveDataSetPayload struct {
sensitiveRecordsSetPayload struct {
Set []*sensitiveDataPayload `json:"set"`
}
@@ -20,10 +21,10 @@ type (
ModuleID uint64 `json:"moduleID,string"`
Module string `json:"module"`
Records []sensitiveData `json:"records"`
Records []sensitiveRecords `json:"records"`
}
sensitiveData struct {
sensitiveRecords struct {
RecordID uint64 `json:"recordID,string"`
Values []map[string]any `json:"values"`
}
@@ -34,7 +35,7 @@ type (
}
privateDataFinder interface {
FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error)
FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error)
}
DataPrivacy struct {
@@ -54,68 +55,51 @@ func (DataPrivacy) New() *DataPrivacy {
}
}
func (ctrl *DataPrivacy) SensitiveDataList(ctx context.Context, r *request.DataPrivacySensitiveDataList) (out interface{}, err error) {
outSet := sensitiveDataSetPayload{}
func (ctrl *DataPrivacy) RecordList(ctx context.Context, r *request.DataPrivacyRecordList) (out interface{}, err error) {
// If we're requesting only specific connections, prepare filter params here
reqConns := make(map[uint64]bool)
hasReqConns := len(r.ConnectionID) > 0
for _, connectionID := range payload.ParseUint64s(r.ConnectionID) {
reqConns[connectionID] = true
}
// All namespaces
namespaces, _, err := ctrl.namespace.Find(ctx, types.NamespaceFilter{})
// Collect sensitive records
ss, err := ctrl.record.FindSensitive(ctx)
if err != nil {
return
}
outSet.Set = make([]*sensitiveDataPayload, 0, 10)
outSet := sensitiveRecordsSetPayload{
Set: make([]*sensitiveDataPayload, 0, 10),
}
for _, n := range namespaces {
// All modules
modules, _, err := ctrl.module.Find(ctx, types.ModuleFilter{NamespaceID: n.ID})
if err != nil {
return nil, err
for _, s := range ss {
// Skip the ones we don't want
if hasReqConns && !reqConns[s.Module.ModelConfig.ConnectionID] {
continue
}
for _, m := range modules {
conn := m.ModelConfig.ConnectionID
if hasReqConns && !reqConns[conn] {
continue
}
sData, err := ctrl.record.FindSensitive(ctx, types.RecordFilter{ModuleID: m.ID, NamespaceID: m.NamespaceID})
if err != nil {
return nil, err
}
if len(sData) == 0 {
continue
}
// Build the payload
payload := &sensitiveDataPayload{
NamespaceID: s.Namespace.ID,
Namespace: s.Namespace.Name,
nsMod := &sensitiveDataPayload{
NamespaceID: n.ID,
Namespace: n.Name,
ModuleID: s.Module.ID,
Module: s.Module.Name,
ModuleID: m.ID,
Module: m.Name,
Records: make([]sensitiveData, 0, len(sData)),
}
for _, a := range sData {
if len(a.Values) == 0 {
continue
}
nsMod.Records = append(nsMod.Records, sensitiveData{
RecordID: a.ID,
Values: a.Values,
})
}
if len(nsMod.Records) == 0 {
continue
}
outSet.Set = append(outSet.Set, nsMod)
Records: make([]sensitiveRecords, 0, len(s.Records)),
}
for _, a := range s.Records {
if len(a.Values) == 0 {
continue
}
payload.Records = append(payload.Records, sensitiveRecords{
RecordID: a.RecordID,
Values: a.Values,
})
}
outSet.Set = append(outSet.Set, payload)
}
return outSet, nil
compose/rest/handlers/dataPrivacy.go
+7 -7
View File
@@ -19,28 +19,28 @@ import (
type (
// Internal API interface
DataPrivacyAPI interface {
SensitiveDataList(context.Context, *request.DataPrivacySensitiveDataList) (interface{}, error)
RecordList(context.Context, *request.DataPrivacyRecordList) (interface{}, error)
ModuleList(context.Context, *request.DataPrivacyModuleList) (interface{}, error)
}
// HTTP API interface
DataPrivacy struct {
SensitiveDataList func(http.ResponseWriter, *http.Request)
ModuleList func(http.ResponseWriter, *http.Request)
RecordList func(http.ResponseWriter, *http.Request)
ModuleList func(http.ResponseWriter, *http.Request)
}
)
func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy {
return &DataPrivacy{
SensitiveDataList: func(w http.ResponseWriter, r *http.Request) {
RecordList: func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
params := request.NewDataPrivacySensitiveDataList()
params := request.NewDataPrivacyRecordList()
if err := params.Fill(r); err != nil {
api.Send(w, r, err)
return
}
value, err := h.SensitiveDataList(r.Context(), params)
value, err := h.RecordList(r.Context(), params)
if err != nil {
api.Send(w, r, err)
return
@@ -70,7 +70,7 @@ func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy {
func (h DataPrivacy) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) {
r.Group(func(r chi.Router) {
r.Use(middlewares...)
r.Get("/data-privacy/sensitive-data", h.SensitiveDataList)
r.Get("/data-privacy/record", h.RecordList)
r.Get("/data-privacy/module", h.ModuleList)
})
}
compose/rest/request/dataPrivacy.go
+8 -8
View File
@@ -33,7 +33,7 @@ var (
type (
// Internal API interface
DataPrivacySensitiveDataList struct {
DataPrivacyRecordList struct {
// SensitivityLevelID GET parameter
//
// Sensitivity Level ID
@@ -68,13 +68,13 @@ type (
}
)
// NewDataPrivacySensitiveDataList request
func NewDataPrivacySensitiveDataList() *DataPrivacySensitiveDataList {
return &DataPrivacySensitiveDataList{}
// NewDataPrivacyRecordList request
func NewDataPrivacyRecordList() *DataPrivacyRecordList {
return &DataPrivacyRecordList{}
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacySensitiveDataList) Auditable() map[string]interface{} {
func (r DataPrivacyRecordList) Auditable() map[string]interface{} {
return map[string]interface{}{
"sensitivityLevelID": r.SensitivityLevelID,
"connectionID": r.ConnectionID,
@@ -82,17 +82,17 @@ func (r DataPrivacySensitiveDataList) Auditable() map[string]interface{} {
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacySensitiveDataList) GetSensitivityLevelID() uint64 {
func (r DataPrivacyRecordList) GetSensitivityLevelID() uint64 {
return r.SensitivityLevelID
}
// Auditable returns all auditable/loggable parameters
func (r DataPrivacySensitiveDataList) GetConnectionID() []string {
func (r DataPrivacyRecordList) GetConnectionID() []string {
return r.ConnectionID
}
// Fill processes request and fills internal variables
func (r *DataPrivacySensitiveDataList) Fill(req *http.Request) (err error) {
func (r *DataPrivacyRecordList) Fill(req *http.Request) (err error) {
{
// GET params
compose/service/module.go
+1 -3
View File
@@ -444,9 +444,7 @@ func (svc module) FindSensitive(ctx context.Context, filter types.PrivacyModuleF
for _, m := range mm {
isPrivate := false
for _, f := range m.Fields {
if !isPrivate {
isPrivate = f.Private
}
isPrivate = isPrivate || f.IsSensitive()
}
if isPrivate && m != nil {
compose/service/record.go
+88 -35
View File
@@ -44,6 +44,9 @@ type (
store store.Storer
namespace namespaceFinder
module moduleFinder
formatter recordValuesFormatter
sanitizer recordValuesSanitizer
validator recordValuesValidator
@@ -88,12 +91,20 @@ type (
recordValueAccessController
}
moduleFinder interface {
Find(ctx context.Context, filter types.ModuleFilter) (set types.ModuleSet, f types.ModuleFilter, err error)
}
namespaceFinder interface {
Find(context.Context, types.NamespaceFilter) (types.NamespaceSet, types.NamespaceFilter, error)
}
RecordService interface {
FindByID(ctx context.Context, namespaceID, moduleID, recordID uint64) (*types.Record, error)
Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error)
Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error)
FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error)
FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error)
RecordExport(context.Context, types.RecordFilter) error
RecordImport(context.Context, error) error
@@ -163,6 +174,9 @@ func Record() *record {
store: DefaultStore,
dal: dal.Service(),
namespace: DefaultNamespace,
module: DefaultModule,
formatter: values.Formatter(),
sanitizer: values.Sanitizer(),
}
@@ -327,53 +341,44 @@ func (svc record) Find(ctx context.Context, filter types.RecordFilter) (set type
return set, f, svc.recordAction(ctx, aProps, RecordActionSearch, err)
}
func (svc record) FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) {
// FindSensitive returns stripped down records for all namespaces/modules where fields define a sensitivity level
func (svc record) FindSensitive(ctx context.Context) (set []types.SensitiveRecordSet, err error) {
var (
m *types.Module
namespaces types.NamespaceSet
modules types.ModuleSet
userID = auth.GetIdentityFromContext(ctx).Identity()
)
err = func() error {
if m, err = loadModule(ctx, svc.store, filter.NamespaceID, filter.ModuleID); err != nil {
return err
}
// Force the query to only show owned records
// @todo allow additional querying
filter.Query = fmt.Sprintf("ownedBy='%d'", auth.GetIdentityFromContext(ctx).Identity())
rr, _, err := svc.Find(ctx, filter)
// Get namespaces
namespaces, _, err = svc.namespace.Find(ctx, types.NamespaceFilter{})
if err != nil {
return err
}
for _, r := range rr {
vv := make([]map[string]any, 0, len(r.Values))
for _, namespace := range namespaces {
// Get corresponding modules
modules, _, err = svc.module.Find(ctx, types.ModuleFilter{NamespaceID: namespace.ID})
if err != nil {
return err
}
for _, f := range m.Fields {
// Skip the ones with no privacy
// @todo allow the request to specify what level we wish to see
if !f.Private {
for _, module := range modules {
// Get sensitive record data
aux, err := svc.findSensitive(ctx, userID, namespace, module, types.RecordFilter{
ModuleID: module.ID,
NamespaceID: namespace.ID,
})
if err != nil {
return err
}
if len(aux.Records) == 0 {
continue
}
values := make([]any, 0, 2)
for _, v := range r.Values.FilterByName(f.Name) {
values = append(values, v.Value)
}
// Make value
vv = append(vv, map[string]any{
"name": f.Name,
"kind": f.Kind,
"isMulti": f.Multi,
"value": values,
})
set = append(set, aux)
}
set = append(set, types.PrivateDataSet{
ID: r.ID,
Values: vv,
})
}
return nil
@@ -382,6 +387,54 @@ func (svc record) FindSensitive(ctx context.Context, filter types.RecordFilter)
return set, err
}
func (svc record) findSensitive(ctx context.Context, userID uint64, namespace *types.Namespace, module *types.Module, filter types.RecordFilter) (out types.SensitiveRecordSet, err error) {
out = types.SensitiveRecordSet{
Namespace: namespace,
Module: module,
}
// Force the query to only show owned records
// @todo allow additional querying
filter.Query = fmt.Sprintf("ownedBy='%d'", userID)
rr, _, err := svc.Find(ctx, filter)
if err != nil {
return
}
for _, r := range rr {
vv := make([]map[string]any, 0, len(r.Values))
for _, f := range module.Fields {
// Skip the ones with no privacy
// @todo allow the request to specify what level we wish to see
if !f.IsSensitive() {
continue
}
values := make([]any, 0, 2)
for _, v := range r.Values.FilterByName(f.Name) {
values = append(values, v.Value)
}
// Make value
vv = append(vv, map[string]any{
"name": f.Name,
"kind": f.Kind,
"isMulti": f.Multi,
"value": values,
})
}
out.Records = append(out.Records, types.SensitiveRecord{
RecordID: r.ID,
Values: vv,
})
}
return
}
func (svc record) RecordImport(ctx context.Context, err error) error {
return svc.recordAction(ctx, &recordActionProps{}, RecordActionImport, err)
}
compose/types/module_field.go
+4
View File
@@ -516,3 +516,7 @@ func (f ModuleField) IsTimeOnly() bool {
func (f ModuleField) IsRef() bool {
return f.Kind == "Record" || f.Kind == "User" || f.Kind == "File"
}
func (f ModuleField) IsSensitive() bool {
return f.Privacy.SensitivityLevel > 0
}
compose/types/record.go
+12 -3
View File
@@ -80,9 +80,18 @@ type (
RecordFilter
}
PrivateDataSet struct {
ID uint64
Values []map[string]any
SensitiveRecord struct {
RecordID uint64
Values []map[string]any
}
SensitiveRecordSet struct {
// Contextual metadata
ConnectionID uint64
Module *Module
Namespace *Namespace
Records []SensitiveRecord
}
)