Shaping tests & post tweaks

pkg/envoy/csv/decoder.go

@@ -103,3 +103,7 @@ func (cr *reader) Next() (map[string]string, error) {
 
 	return mr, nil
 }
+
+func (cr *reader) Count() uint64 {
+	return cr.count
+}

pkg/envoy/csv/decoder_test.go

@@ -0,0 +1,58 @@
+package csv
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/cortezaproject/corteza-server/pkg/envoy"
+	"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
+	"github.com/stretchr/testify/require"
+)
+
+func parseDocument(ctx context.Context, name string) (*resource.ResourceDataset, error) {
+	path := fmt.Sprintf("testdata/%s.csv", name)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	cd := Decoder()
+	ii, err := cd.Decode(ctx, f, &envoy.DecoderOpts{
+		Name: name,
+		Path: path,
+	})
+	if err != nil {
+		return nil, err
+	}
+	rd, _ := ii[0].(*resource.ResourceDataset)
+	return rd, nil
+}
+
+func TestDecoder(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("doc 1", func(t *testing.T) {
+		req := require.New(t)
+
+		ds, err := parseDocument(ctx, "records_1")
+		req.NoError(err)
+		req.NotNil(ds)
+		req.Equal(resource.MakeIdentifiers("records_1"), ds.Identifiers())
+		req.Equal([]string{"id", "c1", "c2", "c3"}, ds.P.Fields())
+		req.Equal(uint64(3), ds.P.Count())
+
+		for i := 0; i < 3; i++ {
+			n, err := ds.P.Next()
+			req.NoError(err)
+			req.NotNil(n)
+
+			req.Len(n, 4)
+		}
+		n, err := ds.P.Next()
+		req.Nil(n)
+		req.Nil(err)
+	})
+}

pkg/envoy/csv/testdata/records_1.csv

@@ -0,0 +1,4 @@
+id,c1,c2,c3
+1,c1.v1,c2.v1,c3.v1
+2,c1.v2,c2.v2,c3.v2
+3,c1.v3,c2.v3,c3.v3

pkg/envoy/resource/dataset.go

@@ -3,6 +3,7 @@ package resource
 type (
 	provider interface {
 		Fields() []string
+		Count() uint64
 		Next() (map[string]string, error)
 	}
 

pkg/envoy/yaml/compose_module.go

@@ -173,7 +173,7 @@ func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error {
 				rt.Key = []string{v.Value}
 			} else {
 				rt.Key = make([]string, 0, 3)
-				eachSeq(v, func(y *yaml.Node) error {
+				eachSeq(v, func(v *yaml.Node) error {
 					rt.Key = append(rt.Key, v.Value)
 					return nil
 				})
@@ -186,7 +186,9 @@ func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error {
 			// Omit cells with empty, /, or - value.
 			if isKind(v, yaml.SequenceNode) {
 				i := uint(0)
-				eachSeq(v, func(y *yaml.Node) error {
+				eachSeq(v, func(v *yaml.Node) error {
+					defer func() { i++ }()
+
 					if v.Value == "" || v.Value == "/" || v.Value == "-" {
 						return nil
 					}
@@ -197,7 +199,6 @@ func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error {
 							Field: v.Value,
 						},
 					})
-					i++
 					return nil
 				})
 			} else if isKind(v, yaml.MappingNode) {

pkg/envoy/yaml/compose_module_test.go

@@ -83,3 +83,62 @@ func TestComposeModule_UnmarshalYAML(t *testing.T) {
 		req.Equal(b.res.Access, rbac.Deny)
 	})
 }
+
+func TestComposeModule_Shaping(t *testing.T) {
+	t.Run("1 key; list mapping", func(t *testing.T) {
+		req := require.New(t)
+
+		doc, err := parseDocument("compose_module_s_1")
+		req.NoError(err)
+		req.NotNil(doc)
+		req.NotNil(doc.compose)
+		req.Len(doc.compose.Modules, 1)
+		m1 := doc.compose.Modules[0]
+
+		tp := m1.recTpl
+		req.NotNil(tp)
+		req.Equal("mod1.csv", tp.Source)
+		req.Equal([]string{"id"}, tp.Key)
+
+		req.Len(tp.Mapping, 2)
+		req.Equal(uint(0), tp.Mapping[0].Index)
+		req.Equal("f1", tp.Mapping[0].Field)
+		req.Equal(uint(3), tp.Mapping[1].Index)
+		req.Equal("f2", tp.Mapping[1].Field)
+	})
+
+	t.Run("2 keys", func(t *testing.T) {
+		req := require.New(t)
+
+		doc, err := parseDocument("compose_module_s_2")
+		req.NoError(err)
+		req.NotNil(doc)
+		req.NotNil(doc.compose)
+		req.Len(doc.compose.Modules, 1)
+		m1 := doc.compose.Modules[0]
+
+		tp := m1.recTpl
+		req.NotNil(tp)
+		req.Equal([]string{"f1", "f2"}, tp.Key)
+	})
+
+	t.Run("complex mapping", func(t *testing.T) {
+		req := require.New(t)
+
+		doc, err := parseDocument("compose_module_s_3")
+		req.NoError(err)
+		req.NotNil(doc)
+		req.NotNil(doc.compose)
+		req.Len(doc.compose.Modules, 1)
+		m1 := doc.compose.Modules[0]
+
+		tp := m1.recTpl
+		req.Len(tp.Mapping, 2)
+		req.Equal(uint(0), tp.Mapping[0].Index)
+		req.Equal("c1", tp.Mapping[0].Cell)
+		req.Equal("f1", tp.Mapping[0].Field)
+		req.Equal(uint(0), tp.Mapping[1].Index)
+		req.Equal("c2", tp.Mapping[1].Cell)
+		req.Equal("f2", tp.Mapping[1].Field)
+	})
+}

pkg/envoy/yaml/testdata/compose_module_s_1.yaml

@@ -0,0 +1,11 @@
+namespace: ns1
+modules:
+  mod1:
+    records:
+      source: mod1.csv
+      key: id
+      mapping:
+        - f1
+        -
+        - /
+        - f2

pkg/envoy/yaml/testdata/compose_module_s_2.yaml

@@ -0,0 +1,7 @@
+namespace: ns1
+modules:
+  mod1:
+    records:
+      key:
+        - f1
+        - f2

pkg/envoy/yaml/testdata/compose_module_s_3.yaml

@@ -0,0 +1,9 @@
+namespace: ns1
+modules:
+  mod1:
+    records:
+      mapping:
+        c1:
+          field: f1
+        c2:
+          field: f2

tests/envoy/main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/cortezaproject/corteza-server/compose/types"
 	"github.com/cortezaproject/corteza-server/pkg/envoy"
+	"github.com/cortezaproject/corteza-server/pkg/envoy/csv"
 	"github.com/cortezaproject/corteza-server/pkg/envoy/directory"
 	"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
 	es "github.com/cortezaproject/corteza-server/pkg/envoy/store"
@@ -62,9 +63,10 @@ func yd(ctx context.Context, suite, fname string) ([]resource.Interface, error)
 
 func dd(ctx context.Context, suite string) ([]resource.Interface, error) {
 	fp := path.Join("testdata", suite)
-	d := yaml.Decoder()
+	yd := yaml.Decoder()
+	cd := csv.Decoder()
 
-	return directory.Decode(ctx, fp, d)
+	return directory.Decode(ctx, fp, yd, cd)
 }
 
 func encode(ctx context.Context, s store.Storer, nn []resource.Interface) error {

tests/envoy/shaping_test.go

@@ -0,0 +1,94 @@
+package envoy
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
+	su "github.com/cortezaproject/corteza-server/pkg/envoy/store"
+	"github.com/cortezaproject/corteza-server/store"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShaping(t *testing.T) {
+	var (
+		ctx    = context.Background()
+		s, err = initStore(ctx)
+	)
+	if err != nil {
+		t.Fatalf("failed to init sqlite in-memory db: %v", err)
+	}
+
+	fullModLoad := func(ctx context.Context, s store.Storer, req *require.Assertions, nsID uint64, handle string) (*types.Module, error) {
+		mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, s, nsID, handle)
+		req.NoError(err)
+		req.NotNil(mod)
+
+		mod.Fields, _, err = store.SearchComposeModuleFields(ctx, s, types.ModuleFieldFilter{ModuleID: []uint64{mod.ID}})
+		req.NoError(err)
+		req.NotNil(mod.Fields)
+		return mod, err
+	}
+
+	ni := uint64(0)
+	su.NextID = func() uint64 {
+		ni++
+		return ni
+	}
+
+	prepare := func(ctx context.Context, s store.Storer, t *testing.T, suite string) (*require.Assertions, error) {
+		req := require.New(t)
+
+		nn, err := dd(ctx, suite)
+		req.NoError(err)
+
+		crs := resource.ComposeRecordShaper()
+		nn, err = resource.Shape(nn, crs)
+		req.NoError(err)
+
+		return req, encode(ctx, s, nn)
+	}
+	// Prepare
+	s, err = initStore(ctx)
+	err = ce(
+		err,
+
+		s.TruncateRoleMembers(ctx),
+		s.TruncateRoles(ctx),
+		s.TruncateComposeNamespaces(ctx),
+		s.TruncateComposeModules(ctx),
+		s.TruncateComposeModuleFields(ctx),
+
+		storeRole(ctx, s, 1, "everyone"),
+		storeRole(ctx, s, 2, "admins"),
+	)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	req, err := prepare(ctx, s, t, "shaping_csv_simple")
+	req.NoError(err)
+
+	t.Run("record shaping", func(t *testing.T) {
+		ns, err := store.LookupComposeNamespaceBySlug(ctx, s, "ns1")
+		req.NotNil(ns)
+		ms, err := fullModLoad(ctx, s, req, ns.ID, "mod1")
+		req.NotNil(ms)
+
+		rr, _, err := store.SearchComposeRecords(ctx, s, ms, types.RecordFilter{})
+		req.NoError(err)
+		req.Len(rr, 2)
+
+		r1 := rr[0]
+		r2 := rr[1]
+
+		req.Len(r1.Values, 2)
+		req.Equal("c1.v1", r1.Values.FilterByName("f1")[0].Value)
+		req.Equal("c2.v1", r1.Values.FilterByName("f2")[0].Value)
+
+		req.Len(r2.Values, 2)
+		req.Equal("c1.v2", r2.Values.FilterByName("f1")[0].Value)
+		req.Equal("c2.v2", r2.Values.FilterByName("f2")[0].Value)
+	})
+}

tests/envoy/testdata/shaping_csv_simple/1100_modules.yaml

@@ -0,0 +1,22 @@
+namespaces:
+  ns1:
+    name: ns1 name
+
+modules:
+  mod1:
+    fields:
+      f1:
+        label: f1 label
+        kind: String
+      f2:
+        label: f2 label
+        kind: String
+
+    records:
+      source: mod1.csv
+      key: id
+      mapping:
+        c1:
+          field: f1
+        c2:
+          field: f2

tests/envoy/testdata/shaping_csv_simple/mod1.csv

@@ -0,0 +1,3 @@
+id,c1,c2
+1,c1.v1,c2.v1
+2,c1.v2,c2.v2