Add POC endpoint for sensitive data collection

compose/rest.yaml

@@ -973,6 +973,19 @@ endpoints:
         type: map[string]interface{}
         parser: parseMapStringInterface
         title: Arguments to pass to the script
+
+- title: Data Privacy
+  entrypoint: dataPrivacy
+  path: "/data-privacy"
+  apis:
+    - name: list sensitive data
+      method: GET
+      title: List sensitive data
+      path: /sensitive-data
+      parameters:
+        get:
+          - { name: sensitivityLevelID, type: "uint64", title: "Sensitivity Level ID", required: false}
+
 - title: Charts
   path: "/namespace/{namespaceID}/chart"
   entrypoint: chart

compose/rest/data_privacy.go

@@ -0,0 +1,96 @@
+package rest
+
+import (
+	"context"
+
+	"github.com/cortezaproject/corteza-server/compose/rest/request"
+	"github.com/cortezaproject/corteza-server/compose/service"
+	"github.com/cortezaproject/corteza-server/compose/types"
+)
+
+type (
+	sensitiveDataSetPayload struct {
+		Set []*sensitiveDataPayload `json:"set"`
+	}
+
+	sensitiveDataPayload struct {
+		NamespaceID uint64 `json:"namespaceID"`
+		Namespace   string `json:"namespace"`
+		ModuleID    uint64 `json:"moduleID"`
+		Module      string `json:"module"`
+
+		Records []sensitiveData `json:"records"`
+	}
+
+	sensitiveData struct {
+		RecordID uint64           `json:"recordID"`
+		Values   []map[string]any `json:"values"`
+	}
+
+	privateDataFinder interface {
+		FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error)
+	}
+
+	DataPrivacy struct {
+		record    privateDataFinder
+		module    service.ModuleService
+		namespace service.NamespaceService
+	}
+)
+
+func (DataPrivacy) New() *DataPrivacy {
+	return &DataPrivacy{
+		record:    service.DefaultRecord,
+		module:    service.DefaultModule,
+		namespace: service.DefaultNamespace,
+	}
+}
+
+func (ctrl *DataPrivacy) ListSensitiveData(ctx context.Context, r *request.DataPrivacyListSensitiveData) (out interface{}, err error) {
+	outSet := sensitiveDataSetPayload{}
+
+	// All namespaces
+	namespaces, _, err := ctrl.namespace.Find(ctx, types.NamespaceFilter{})
+	if err != nil {
+		return
+	}
+
+	outSet.Set = make([]*sensitiveDataPayload, 0, 10)
+
+	for _, n := range namespaces {
+		// All modules
+		modules, _, err := ctrl.module.Find(ctx, types.ModuleFilter{NamespaceID: n.ID})
+		if err != nil {
+			return nil, err
+		}
+		for _, m := range modules {
+			if m.Privacy.SensitivityLevel == 0 {
+				continue
+			}
+
+			sData, err := ctrl.record.FindSensitive(ctx, types.RecordFilter{ModuleID: m.ID, NamespaceID: m.NamespaceID})
+			if err != nil {
+				return nil, err
+			}
+
+			nsMod := &sensitiveDataPayload{
+				NamespaceID: n.ID,
+				Namespace:   n.Name,
+
+				ModuleID: m.ID,
+				Module:   m.Name,
+
+				Records: make([]sensitiveData, 0, len(sData)),
+			}
+			for _, a := range sData {
+				nsMod.Records = append(nsMod.Records, sensitiveData{
+					RecordID: a.ID,
+					Values:   a.Values,
+				})
+			}
+			outSet.Set = append(outSet.Set, nsMod)
+		}
+	}
+
+	return outSet, nil
+}

compose/rest/handlers/dataPrivacy.go

@@ -0,0 +1,57 @@
+package handlers
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+// Definitions file that controls how this file is generated:
+//
+
+import (
+	"context"
+	"github.com/cortezaproject/corteza-server/compose/rest/request"
+	"github.com/cortezaproject/corteza-server/pkg/api"
+	"github.com/go-chi/chi/v5"
+	"net/http"
+)
+
+type (
+	// Internal API interface
+	DataPrivacyAPI interface {
+		ListSensitiveData(context.Context, *request.DataPrivacyListSensitiveData) (interface{}, error)
+	}
+
+	// HTTP API interface
+	DataPrivacy struct {
+		ListSensitiveData func(http.ResponseWriter, *http.Request)
+	}
+)
+
+func NewDataPrivacy(h DataPrivacyAPI) *DataPrivacy {
+	return &DataPrivacy{
+		ListSensitiveData: func(w http.ResponseWriter, r *http.Request) {
+			defer r.Body.Close()
+			params := request.NewDataPrivacyListSensitiveData()
+			if err := params.Fill(r); err != nil {
+				api.Send(w, r, err)
+				return
+			}
+
+			value, err := h.ListSensitiveData(r.Context(), params)
+			if err != nil {
+				api.Send(w, r, err)
+				return
+			}
+
+			api.Send(w, r, value)
+		},
+	}
+}
+
+func (h DataPrivacy) MountRoutes(r chi.Router, middlewares ...func(http.Handler) http.Handler) {
+	r.Group(func(r chi.Router) {
+		r.Use(middlewares...)
+		r.Get("/data-privacy/sensitive-data", h.ListSensitiveData)
+	})
+}

compose/rest/request/dataPrivacy.go

@@ -0,0 +1,77 @@
+package request
+
+// This file is auto-generated.
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+//
+// Definitions file that controls how this file is generated:
+//
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/cortezaproject/corteza-server/pkg/payload"
+	"github.com/go-chi/chi/v5"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strings"
+)
+
+// dummy vars to prevent
+// unused imports complain
+var (
+	_ = chi.URLParam
+	_ = multipart.ErrMessageTooLarge
+	_ = payload.ParseUint64s
+	_ = strings.ToLower
+	_ = io.EOF
+	_ = fmt.Errorf
+	_ = json.NewEncoder
+)
+
+type (
+	// Internal API interface
+	DataPrivacyListSensitiveData struct {
+		// SensitivityLevelID GET parameter
+		//
+		// Sensitivity Level ID
+		SensitivityLevelID uint64 `json:",string"`
+	}
+)
+
+// NewDataPrivacyListSensitiveData request
+func NewDataPrivacyListSensitiveData() *DataPrivacyListSensitiveData {
+	return &DataPrivacyListSensitiveData{}
+}
+
+// Auditable returns all auditable/loggable parameters
+func (r DataPrivacyListSensitiveData) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"sensitivityLevelID": r.SensitivityLevelID,
+	}
+}
+
+// Auditable returns all auditable/loggable parameters
+func (r DataPrivacyListSensitiveData) GetSensitivityLevelID() uint64 {
+	return r.SensitivityLevelID
+}
+
+// Fill processes request and fills internal variables
+func (r *DataPrivacyListSensitiveData) Fill(req *http.Request) (err error) {
+
+	{
+		// GET params
+		tmp := req.URL.Query()
+
+		if val, ok := tmp["sensitivityLevelID"]; ok && len(val) > 0 {
+			r.SensitivityLevelID, err = payload.ParseUint64(val[0]), nil
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return err
+}

compose/rest/router.go

@@ -18,6 +18,7 @@ func MountRoutes() func(r chi.Router) {
 			notification = Notification{}.New()
 			attachment   = Attachment{}.New()
 			automation   = Automation{}.New()
+			dataPrivacy  = DataPrivacy{}.New()
 		)
 
 		// Initialize handlers & controllers.
@@ -38,6 +39,7 @@ func MountRoutes() func(r chi.Router) {
 			handlers.NewRecord(record).MountRoutes(r)
 			handlers.NewChart(chart).MountRoutes(r)
 			handlers.NewNotification(notification).MountRoutes(r)
+			handlers.NewDataPrivacy(dataPrivacy).MountRoutes(r)
 		})
 	}
 }

compose/service/record.go

@@ -94,6 +94,7 @@ type (
 
 		Report(ctx context.Context, namespaceID, moduleID uint64, metrics, dimensions, filter string) (interface{}, error)
 		Find(ctx context.Context, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error)
+		FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error)
 		RecordExport(context.Context, types.RecordFilter) error
 		RecordImport(context.Context, error) error
 
@@ -334,6 +335,61 @@ func (svc record) Find(ctx context.Context, filter types.RecordFilter) (set type
 	return set, f, svc.recordAction(ctx, aProps, RecordActionSearch, err)
 }
 
+func (svc record) FindSensitive(ctx context.Context, filter types.RecordFilter) (set []types.PrivateDataSet, err error) {
+	var (
+		m *types.Module
+	)
+
+	err = func() error {
+		if m, err = loadModule(ctx, svc.store, filter.ModuleID); err != nil {
+			return err
+		}
+
+		// Force the query to only show owned records
+		// @todo allow additional querying
+		filter.Query = fmt.Sprintf("ownedBy='%d'", auth.GetIdentityFromContext(ctx).Identity())
+
+		rr, _, err := svc.Find(ctx, filter)
+		if err != nil {
+			return err
+		}
+
+		for _, r := range rr {
+			vv := make([]map[string]any, 0, len(r.Values))
+
+			for _, f := range m.Fields {
+				// Skip the ones with no privacy
+				// @todo allow the request to specify what level we wish to see
+				if f.Privacy.SensitivityLevel == 0 {
+					continue
+				}
+
+				values := make([]any, 0, 2)
+				for _, v := range r.Values.FilterByName(f.Name) {
+					values = append(values, v.Value)
+				}
+
+				// Make value
+				vv = append(vv, map[string]any{
+					"name":    f.Name,
+					"kind":    f.Kind,
+					"isMulti": f.Multi,
+					"value":   values,
+				})
+			}
+
+			set = append(set, types.PrivateDataSet{
+				ID:     r.ID,
+				Values: vv,
+			})
+		}
+
+		return nil
+	}()
+
+	return set, err
+}
+
 func (svc record) RecordImport(ctx context.Context, err error) error {
 	return svc.recordAction(ctx, &recordActionProps{}, RecordActionImport, err)
 }

compose/types/record.go

@@ -79,6 +79,11 @@ type (
 		constraints map[string][]any
 		RecordFilter
 	}
+
+	PrivateDataSet struct {
+		ID     uint64
+		Values []map[string]any
+	}
 )
 
 const (