From 53cf96848ab5182e11ad6c392bdea3a4b45c4d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Mon, 10 May 2021 10:51:50 +0200 Subject: [PATCH] Tweak ComposeRecord exporting * Fix pagination cursors * Use service functions for AC * Add tests --- compose/service/record.go | 24 +++++----- pkg/envoy/store/compose.go | 16 +++++-- tests/compose/record_test.go | 57 +++++++++++++++++++++++- tests/envoy/compose.go | 32 ++++++++++++++ tests/envoy/store_csv_records_test.go | 59 +++++++++++++++++++++++++ tests/envoy/store_jsonl_records_test.go | 59 +++++++++++++++++++++++++ 6 files changed, 232 insertions(+), 15 deletions(-) diff --git a/compose/service/record.go b/compose/service/record.go index 9779da1e9..b9c6ae012 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -226,7 +226,7 @@ func (svc record) lookup(ctx context.Context, namespaceID, moduleID uint64, look return RecordErrNotAllowedToRead() } - trimUnreadableRecordFields(ctx, svc.ac, m, r) + ComposeRecordFilterAC(ctx, svc.ac, m, r) if err = label.Load(ctx, svc.store, r); err != nil { return err @@ -281,13 +281,7 @@ func (svc record) Find(ctx context.Context, filter types.RecordFilter) (set type return err } - filter.Check = func(res *types.Record) (bool, error) { - if !svc.ac.CanReadRecord(ctx, m) { - return false, nil - } - - return true, nil - } + filter.Check = ComposeRecordFilterChecker(ctx, svc.ac, m) if len(filter.Labels) > 0 { filter.LabeledIDs, err = label.Search( @@ -321,7 +315,7 @@ func (svc record) Find(ctx context.Context, filter types.RecordFilter) (set type return nil }) - trimUnreadableRecordFields(ctx, svc.ac, m, set...) + ComposeRecordFilterAC(ctx, svc.ac, m, set...) return nil }() @@ -1353,8 +1347,18 @@ func (svc record) Iterator(ctx context.Context, f types.RecordFilter, fn eventbu } +func ComposeRecordFilterChecker(ctx context.Context, ac recordAccessController, m *types.Module) func(*types.Record) (bool, error) { + return func(res *types.Record) (bool, error) { + if !ac.CanReadRecord(ctx, m) { + return false, nil + } + + return true, nil + } +} + // checks record-value-read access permissions for all module fields and removes unreadable fields from all records -func trimUnreadableRecordFields(ctx context.Context, ac recordValueAccessController, m *types.Module, rr ...*types.Record) { +func ComposeRecordFilterAC(ctx context.Context, ac recordValueAccessController, m *types.Module, rr ...*types.Record) { var ( readableFields = map[string]bool{} ) diff --git a/pkg/envoy/store/compose.go b/pkg/envoy/store/compose.go index d4268a1c5..d9b1b08b0 100644 --- a/pkg/envoy/store/compose.go +++ b/pkg/envoy/store/compose.go @@ -5,10 +5,12 @@ import ( "strconv" "strings" + "github.com/cortezaproject/corteza-server/compose/service" "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/envoy" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/rbac" "github.com/cortezaproject/corteza-server/store" stypes "github.com/cortezaproject/corteza-server/system/types" ) @@ -165,6 +167,8 @@ func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer } } + ac := service.AccessControl(rbac.Global()) + if len(d.namespaceID) > 0 { ffNs := make([]*composeRecordFilter, 0, len(ff)+len(d.namespaceID)) for _, nsID := range d.namespaceID { @@ -232,6 +236,8 @@ func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer } mod.Fields = ff + aux.Check = service.ComposeRecordFilterChecker(ctx, ac, mod) + // Refs auxRecord := &composeRecordAux{ refMod: strconv.FormatUint(f.ModuleID, 10), @@ -242,17 +248,19 @@ func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer // Walker auxRecord.walker = func(cb func(r *resource.ComposeRecordRaw) error) error { - var nn types.RecordSet + var rr types.RecordSet var fn types.RecordFilter var err error for { - nn, fn, err = s.SearchComposeRecords(ctx, mod, types.RecordFilter(aux)) + rr, fn, err = s.SearchComposeRecords(ctx, mod, types.RecordFilter(aux)) if err != nil { return err } - for _, n := range nn { + service.ComposeRecordFilterAC(ctx, ac, mod, rr...) + + for _, n := range rr { // Create a raw record r := &resource.ComposeRecordRaw{ ID: strconv.FormatUint(n.ID, 10), @@ -267,7 +275,7 @@ func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer } } - if f.NextPage != nil { + if fn.NextPage != nil { aux.PageCursor = fn.NextPage } else { break diff --git a/tests/compose/record_test.go b/tests/compose/record_test.go index 79ee17477..32707edc6 100644 --- a/tests/compose/record_test.go +++ b/tests/compose/record_test.go @@ -135,8 +135,13 @@ func (h helper) repoMakeRecordModuleWithFieldsRequired(name string, ff ...*types } func (h helper) makeRecord(module *types.Module, rvs ...*types.RecordValue) *types.Record { + recID := id.Next() + for _, rv := range rvs { + rv.RecordID = recID + } + rec := &types.Record{ - ID: id.Next(), + ID: recID, CreatedAt: time.Now(), ModuleID: module.ID, NamespaceID: module.NamespaceID, @@ -181,9 +186,59 @@ func TestRecordList(t *testing.T) { h.apiInit(). Get(fmt.Sprintf("/namespace/%d/module/%d/record/", module.NamespaceID, module.ID)). + Query("incTotal", "true"). Expect(t). Status(http.StatusOK). Assert(helpers.AssertNoErrors). + Assert(jsonpath.Equal(`$.response.filter.total`, float64(2))). + End() +} + +func TestRecordListForbidenRecords(t *testing.T) { + h := newHelper(t) + h.clearRecords() + + module := h.repoMakeRecordModuleWithFields("record testing module") + h.deny(types.ModuleRBACResource.AppendWildcard(), "record.read") + + h.makeRecord(module) + h.makeRecord(module) + + h.apiInit(). + Get(fmt.Sprintf("/namespace/%d/module/%d/record/", module.NamespaceID, module.ID)). + Query("incTotal", "true"). + Expect(t). + Status(http.StatusOK). + Assert(helpers.AssertNoErrors). + // not present because omitted when empty + Assert(jsonpath.NotPresent(`$.response.filter.total`)). + End() +} + +func TestRecordListForbidenFields(t *testing.T) { + h := newHelper(t) + h.clearRecords() + + module := h.repoMakeRecordModuleWithFields("record testing module") + h.deny(types.ModuleFieldRBACResource.AppendID(module.Fields[0].ID), "record.value.read") + + h.makeRecord(module, &types.RecordValue{Name: "name", Value: "v_name_0"}, &types.RecordValue{Name: "email", Value: "v_email_0"}) + h.makeRecord(module, &types.RecordValue{Name: "name", Value: "v_name_1"}, &types.RecordValue{Name: "email", Value: "v_email_1"}) + + h.apiInit(). + Get(fmt.Sprintf("/namespace/%d/module/%d/record/", module.NamespaceID, module.ID)). + Query("incTotal", "true"). + Expect(t). + Status(http.StatusOK). + Assert(helpers.AssertNoErrors). + Assert(jsonpath.Equal(`$.response.filter.total`, float64(2))). + Assert(jsonpath.Len(`$.response.set`, 2)). + // rec 1 + Assert(jsonpath.Len(`$.response.set[0].values`, 1)). + Assert(jsonpath.Equal(`$.response.set[0].values[0].name`, "email")). + // rec 2 + Assert(jsonpath.Len(`$.response.set[1].values`, 1)). + Assert(jsonpath.Equal(`$.response.set[1].values[0].name`, "email")). End() } diff --git a/tests/envoy/compose.go b/tests/envoy/compose.go index f5460b2c3..535239660 100644 --- a/tests/envoy/compose.go +++ b/tests/envoy/compose.go @@ -284,3 +284,35 @@ func sTestComposeRecord(ctx context.Context, t *testing.T, s store.Storer, nsID, return rec } + +func sTestComposeRecordRaw(ctx context.Context, t *testing.T, s store.Storer, nsID, modID, usrID uint64, vv ...*types.RecordValue) *types.Record { + recID := su.NextID() + for _, v := range vv { + v.RecordID = recID + } + + rec := &types.Record{ + ID: recID, + NamespaceID: nsID, + ModuleID: modID, + Values: vv, + + CreatedAt: createdAt, + UpdatedAt: &updatedAt, + OwnedBy: usrID, + CreatedBy: usrID, + UpdatedBy: usrID, + } + + mod := &types.Module{ + ID: modID, + NamespaceID: nsID, + } + + err := store.CreateComposeRecord(ctx, s, mod, rec) + if err != nil { + t.Fatal(err) + } + + return rec +} diff --git a/tests/envoy/store_csv_records_test.go b/tests/envoy/store_csv_records_test.go index 6786e1098..d672655ef 100644 --- a/tests/envoy/store_csv_records_test.go +++ b/tests/envoy/store_csv_records_test.go @@ -2,6 +2,7 @@ package envoy import ( "context" + "fmt" "strconv" "testing" "time" @@ -12,6 +13,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/envoy/csv" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" su "github.com/cortezaproject/corteza-server/pkg/envoy/store" + "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" "github.com/stretchr/testify/require" ) @@ -95,6 +97,63 @@ func TestStoreCsv_records(t *testing.T) { req.Equal("10", vv[0].Value) }, }, + + { + name: "paged", + pre: func(ctx context.Context, s store.Storer) (error, *su.DecodeFilter) { + truncateStore(ctx, s, t) + + ns := sTestComposeNamespace(ctx, t, s, "base") + mod := sTestComposeModule(ctx, t, s, ns.ID, "base") + usr := sTestUser(ctx, t, s, "base") + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "10"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "11"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "12"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "13"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "14"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + + df := su.NewDecodeFilter(). + ComposeRecord(&types.RecordFilter{ + NamespaceID: ns.ID, + ModuleID: mod.ID, + Paging: filter.Paging{ + Limit: 2, + }, + }) + return nil, df + }, + check: func(ctx context.Context, s store.Storer, req *require.Assertions) { + ns, err := store.LookupComposeNamespaceBySlug(ctx, s, "base_namespace") + req.NoError(err) + mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, s, ns.ID, "base_module") + req.NoError(err) + usr, err := store.LookupUserByHandle(ctx, s, "base_user") + req.NoError(err) + _ = usr + + rr, _, err := store.SearchComposeRecords(ctx, s, mod, types.RecordFilter{ + ModuleID: mod.ID, + NamespaceID: ns.ID, + }) + req.NoError(err) + req.Len(rr, 5) + + for i, rec := range rr { + req.Equal(ns.ID, rec.NamespaceID) + req.Equal(mod.ID, rec.ModuleID) + + req.Equal(createdAt.Format(time.RFC3339), rec.CreatedAt.Format(time.RFC3339)) + req.Equal(updatedAt.Format(time.RFC3339), rec.UpdatedAt.Format(time.RFC3339)) + req.Equal(usr.ID, rec.OwnedBy) + req.Equal(usr.ID, rec.CreatedBy) + req.Equal(usr.ID, rec.UpdatedBy) + + vv := rec.Values.FilterByName("module_field_number") + req.Len(vv, 1) + req.Equal(fmt.Sprintf("1%d", i), vv[0].Value) + } + }, + }, } for _, c := range cases { diff --git a/tests/envoy/store_jsonl_records_test.go b/tests/envoy/store_jsonl_records_test.go index f93b352d1..22348aaca 100644 --- a/tests/envoy/store_jsonl_records_test.go +++ b/tests/envoy/store_jsonl_records_test.go @@ -2,6 +2,7 @@ package envoy import ( "context" + "fmt" "strconv" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/envoy/json" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" su "github.com/cortezaproject/corteza-server/pkg/envoy/store" + "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" "github.com/stretchr/testify/require" ) @@ -95,6 +97,63 @@ func TestStoreJsonl_records(t *testing.T) { req.Equal("10", vv[0].Value) }, }, + + { + name: "paged", + pre: func(ctx context.Context, s store.Storer) (error, *su.DecodeFilter) { + truncateStore(ctx, s, t) + + ns := sTestComposeNamespace(ctx, t, s, "base") + mod := sTestComposeModule(ctx, t, s, ns.ID, "base") + usr := sTestUser(ctx, t, s, "base") + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "10"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "11"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "12"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "13"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + sTestComposeRecordRaw(ctx, t, s, ns.ID, mod.ID, usr.ID, &types.RecordValue{Name: "module_field_number", Value: "14"}, &types.RecordValue{Name: "module_field_string", Value: "v"}) + + df := su.NewDecodeFilter(). + ComposeRecord(&types.RecordFilter{ + NamespaceID: ns.ID, + ModuleID: mod.ID, + Paging: filter.Paging{ + Limit: 2, + }, + }) + return nil, df + }, + check: func(ctx context.Context, s store.Storer, req *require.Assertions) { + ns, err := store.LookupComposeNamespaceBySlug(ctx, s, "base_namespace") + req.NoError(err) + mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, s, ns.ID, "base_module") + req.NoError(err) + usr, err := store.LookupUserByHandle(ctx, s, "base_user") + req.NoError(err) + _ = usr + + rr, _, err := store.SearchComposeRecords(ctx, s, mod, types.RecordFilter{ + ModuleID: mod.ID, + NamespaceID: ns.ID, + }) + req.NoError(err) + req.Len(rr, 5) + + for i, rec := range rr { + req.Equal(ns.ID, rec.NamespaceID) + req.Equal(mod.ID, rec.ModuleID) + + req.Equal(createdAt.Format(time.RFC3339), rec.CreatedAt.Format(time.RFC3339)) + req.Equal(updatedAt.Format(time.RFC3339), rec.UpdatedAt.Format(time.RFC3339)) + req.Equal(usr.ID, rec.OwnedBy) + req.Equal(usr.ID, rec.CreatedBy) + req.Equal(usr.ID, rec.UpdatedBy) + + vv := rec.Values.FilterByName("module_field_number") + req.Len(vv, 1) + req.Equal(fmt.Sprintf("1%d", i), vv[0].Value) + } + }, + }, } for _, c := range cases {