From 4d9a2d01817e07906152eb8a7dea1ca6c0dbd68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jun 2022 13:57:37 +0200 Subject: [PATCH] Add POC endpoint for sensitive data collection --- compose/rest.yaml | 13 ++++ compose/rest/data_privacy.go | 96 ++++++++++++++++++++++++++++ compose/rest/handlers/dataPrivacy.go | 57 +++++++++++++++++ compose/rest/request/dataPrivacy.go | 77 ++++++++++++++++++++++ compose/rest/router.go | 2 + compose/service/record.go | 56 ++++++++++++++++ compose/types/record.go | 5 ++ 7 files changed, 306 insertions(+) create mode 100644 compose/rest/data_privacy.go create mode 100644 compose/rest/handlers/dataPrivacy.go create mode 100644 compose/rest/request/dataPrivacy.go diff --git a/compose/rest.yaml b/compose/rest.yaml index 3fe18c0af..9ef1400ab 100644 --- a/compose/rest.yaml +++ b/compose/rest.yaml @@ -973,6 +973,19 @@ endpoints: type: map[string]interface{} parser: parseMapStringInterface title: Arguments to pass to the script + +- title: Data Privacy + entrypoint: dataPrivacy + path: "/data-privacy" + apis: + - name: list sensitive data + method: GET + title: List sensitive data + path: /sensitive-data + parameters: + get: + - { name: sensitivityLevelID, type: "uint64", title: "Sensitivity Level ID", required: false} + - title: Charts path: "/namespace/{namespaceID}/chart" entrypoint: chart diff --git a/compose/rest/data_privacy.go b/compose/rest/data_privacy.go new file mode 100644 index 000000000..4901c7614 --- /dev/null +++ b/compose/rest/data_privacy.go @@ -0,0 +1,96 @@ +package rest + +import ( + "context" + + "github.com/cortezaproject/corteza-server/compose/rest/request" + "github.com/cortezaproject/corteza-server/compose/service" + "github.com/cortezaproject/corteza-server/compose/types" +) + +type ( + sensitiveDataSetPayload struct { + Set []*sensitiveDataPayload `json:"set"` + } + + sensitiveDataPayload struct { + NamespaceID uint64 `json:"namespaceID"` + Namespace string `json:"namespace"` + ModuleID uint64 `json:"moduleID"` + Module string `json:"module"` + + Records []sensitiveData `json:"records"` + } + + sensitiveData struct { + RecordID uint64 `json:"recordID"` + Values []map[string]any `json:"values"` + } + + privateDataFinder interface { + FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) + } + + DataPrivacy struct { + record privateDataFinder + module service.ModuleService + namespace service.NamespaceService + } +) + +func (DataPrivacy) New() *DataPrivacy { + return &DataPrivacy{ + record: service.DefaultRecord, + module: service.DefaultModule, + namespace: service.DefaultNamespace, + } +} + +func (ctrl *DataPrivacy) ListSensitiveData(ctx context.Context, r *request.DataPrivacyListSensitiveData) (out interface{}, err error) { + outSet := sensitiveDataSetPayload{} + + // All namespaces + namespaces, _, err := ctrl.namespace.Find(ctx, types.NamespaceFilter{}) + if err != nil { + return + } + + outSet.Set = make([]*sensitiveDataPayload, 0, 10) + + for _, n := range namespaces { + // All modules + modules, _, err := ctrl.module.Find(ctx, types.ModuleFilter{NamespaceID: n.ID}) + if err != nil { + return nil, err + } + for _, m := range modules { + if m.Privacy.SensitivityLevel == 0 { + continue + } + + sData, err := ctrl.record.FindSensitive(ctx, types.RecordFilter{ModuleID: m.ID, NamespaceID: m.NamespaceID}) + if err != nil { + return nil, err + } + + nsMod := &sensitiveDataPayload{ + NamespaceID: n.ID, + Namespace: n.Name, + + ModuleID: m.ID, + Module: m.Name, + + Records: make([]sensitiveData, 0, len(sData)), + } + for _, a := range sData { + nsMod.Records = append(nsMod.Records, sensitiveData{ + RecordID: a.ID, + Values: a.Values, + }) + } + outSet.Set = append(outSet.Set, nsMod) + } + } + + return outSet, nil +} diff --git a/compose/rest/handlers/dataPrivacy.go b/compose/rest/handlers/dataPrivacy.go new file mode 100644 index 000000000..36075e550 --- /dev/null +++ b/compose/rest/handlers/dataPrivacy.go @@ -0,0 +1,57 @@ +package handlers + +// This file is auto-generated. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// Definitions file that controls how this file is generated: +// + +import ( + "context" + "github.com/cortezaproject/corteza-server/compose/rest/request" + "github.com/cortezaproject/corteza-server/pkg/api" + "github.com/go-chi/chi/v5" + "net/http" +) + +type ( + // Internal API interface + DataPrivacyAPI interface { + ListSensitiveData(context.Context, *request.DataPrivacyListSensitiveData) (interface{}, error) + } + + // HTTP API interface + DataPrivacy struct { + ListSensitiveData func(http.ResponseWriter, *http.Request) + } +) + +func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy { + return &DataPrivacy{ + ListSensitiveData: func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + params := request.NewDataPrivacyListSensitiveData() + if err := params.Fill(r); err != nil { + api.Send(w, r, err) + return + } + + value, err := h.ListSensitiveData(r.Context(), params) + if err != nil { + api.Send(w, r, err) + return + } + + api.Send(w, r, value) + }, + } +} + +func (h DataPrivacy) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) { + r.Group(func(r chi.Router) { + r.Use(middlewares...) + r.Get("/data-privacy/sensitive-data", h.ListSensitiveData) + }) +} diff --git a/compose/rest/request/dataPrivacy.go b/compose/rest/request/dataPrivacy.go new file mode 100644 index 000000000..d16f7ab2d --- /dev/null +++ b/compose/rest/request/dataPrivacy.go @@ -0,0 +1,77 @@ +package request + +// This file is auto-generated. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// Definitions file that controls how this file is generated: +// + +import ( + "encoding/json" + "fmt" + "github.com/cortezaproject/corteza-server/pkg/payload" + "github.com/go-chi/chi/v5" + "io" + "mime/multipart" + "net/http" + "strings" +) + +// dummy vars to prevent +// unused imports complain +var ( + _ = chi.URLParam + _ = multipart.ErrMessageTooLarge + _ = payload.ParseUint64s + _ = strings.ToLower + _ = io.EOF + _ = fmt.Errorf + _ = json.NewEncoder +) + +type ( + // Internal API interface + DataPrivacyListSensitiveData struct { + // SensitivityLevelID GET parameter + // + // Sensitivity Level ID + SensitivityLevelID uint64 `json:",string"` + } +) + +// NewDataPrivacyListSensitiveData request +func NewDataPrivacyListSensitiveData() *DataPrivacyListSensitiveData { + return &DataPrivacyListSensitiveData{} +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyListSensitiveData) Auditable() map[string]interface{} { + return map[string]interface{}{ + "sensitivityLevelID": r.SensitivityLevelID, + } +} + +// Auditable returns all auditable/loggable parameters +func (r DataPrivacyListSensitiveData) GetSensitivityLevelID() uint64 { + return r.SensitivityLevelID +} + +// Fill processes request and fills internal variables +func (r *DataPrivacyListSensitiveData) Fill(req *http.Request) (err error) { + + { + // GET params + tmp := req.URL.Query() + + if val, ok := tmp["sensitivityLevelID"]; ok && len(val) > 0 { + r.SensitivityLevelID, err = payload.ParseUint64(val[0]), nil + if err != nil { + return err + } + } + } + + return err +} diff --git a/compose/rest/router.go b/compose/rest/router.go index fb0bce59c..b29fca55e 100644 --- a/compose/rest/router.go +++ b/compose/rest/router.go @@ -18,6 +18,7 @@ func MountRoutes() func(r chi.Router) { notification = Notification{}.New() attachment = Attachment{}.New() automation = Automation{}.New() + dataPrivacy = DataPrivacy{}.New() ) // Initialize handlers & controllers. @@ -38,6 +39,7 @@ func MountRoutes() func(r chi.Router) { handlers.NewRecord(record).MountRoutes(r) handlers.NewChart(chart).MountRoutes(r) handlers.NewNotification(notification).MountRoutes(r) + handlers.NewDataPrivacy(dataPrivacy).MountRoutes(r) }) } } diff --git a/compose/service/record.go b/compose/service/record.go index fa9442f40..ea1093d29 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -94,6 +94,7 @@ type ( Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error) Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) + FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) RecordExport(context.Context, types.RecordFilter) error RecordImport(context.Context, error) error @@ -334,6 +335,61 @@ func (svc record) Find(ctx context.Context, filter types.RecordFilter) (set type return set, f, svc.recordAction(ctx, aProps, RecordActionSearch, err) } +func (svc record) FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) { + var ( + m *types.Module + ) + + err = func() error { + if m, err = loadModule(ctx, svc.store, filter.ModuleID); err != nil { + return err + } + + // Force the query to only show owned records + // @todo allow additional querying + filter.Query = fmt.Sprintf("ownedBy='%d'", auth.GetIdentityFromContext(ctx).Identity()) + + rr, _, err := svc.Find(ctx, filter) + if err != nil { + return err + } + + for _, r := range rr { + vv := make([]map[string]any, 0, len(r.Values)) + + for _, f := range m.Fields { + // Skip the ones with no privacy + // @todo allow the request to specify what level we wish to see + if f.Privacy.SensitivityLevel == 0 { + continue + } + + values := make([]any, 0, 2) + for _, v := range r.Values.FilterByName(f.Name) { + values = append(values, v.Value) + } + + // Make value + vv = append(vv, map[string]any{ + "name": f.Name, + "kind": f.Kind, + "isMulti": f.Multi, + "value": values, + }) + } + + set = append(set, types.PrivateDataSet{ + ID: r.ID, + Values: vv, + }) + } + + return nil + }() + + return set, err +} + func (svc record) RecordImport(ctx context.Context, err error) error { return svc.recordAction(ctx, &recordActionProps{}, RecordActionImport, err) } diff --git a/compose/types/record.go b/compose/types/record.go index 9146d5545..b83ffa2f0 100644 --- a/compose/types/record.go +++ b/compose/types/record.go @@ -79,6 +79,11 @@ type ( constraints map[string][]any RecordFilter } + + PrivateDataSet struct { + ID uint64 + Values []map[string]any + } ) const (