3
0

Implement data shaping

Unstructured datasources (csv, jsonl, ...) are defined as resource
datasets.
These datasets are then "shaped" based on some template.

The result is an envoy resource like any other.
This commit is contained in:
Tomaž Jerman
2020-11-26 18:39:50 +01:00
parent 7ca422bdd3
commit 10b300328d
17 changed files with 543 additions and 26 deletions
+4 -4
View File
@@ -7,7 +7,7 @@ import (
)
type (
graphBuilder struct {
builder struct {
pp []Preparer
}
@@ -31,13 +31,13 @@ type (
}
)
func NewGraphBuilder(pp ...Preparer) *graphBuilder {
return &graphBuilder{
func NewBuilder(pp ...Preparer) *builder {
return &builder{
pp: pp,
}
}
func (b *graphBuilder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) {
func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) {
g := newGraph()
// Prepare nodes for all resources
+5 -5
View File
@@ -33,7 +33,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
ctx := context.Background()
t.Run("single, simple node; a", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
rr := []resource.Interface{
&testResource{
@@ -54,7 +54,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
})
t.Run("simple node link; a -> b", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
rr := []resource.Interface{
&testResource{
@@ -84,7 +84,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
})
t.Run("cyclic node link; a -> b -> a", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
rr := []resource.Interface{
&testResource{
@@ -118,7 +118,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
})
t.Run("node with missing dep; a -> nill", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
rr := []resource.Interface{
&testResource{
@@ -139,7 +139,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
})
t.Run("self-cycle; a -> a", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
rr := []resource.Interface{
&testResource{
+105
View File
@@ -0,0 +1,105 @@
package csv
import (
"bytes"
"context"
"encoding/csv"
"io"
"os"
"path/filepath"
"strings"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
)
type (
// wrapper struct for csv related methods
decoder struct{}
// csv decoder wrapper for additional bits
reader struct {
c *csv.Reader
header []string
count uint64
}
)
// Decoder initializes and returns a fresh CSV decoder
func Decoder() *decoder {
return &decoder{}
}
// CanDecodeFile determines if the file can be determined by this decoder
func (y *decoder) CanDecodeFile(i os.FileInfo) bool {
return strings.Trim(filepath.Ext(i.Name()), ".") == "csv"
}
// Decode decodes the given io.Reader into a generic resource dataset
func (c *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts) ([]resource.Interface, error) {
cr := &reader{}
var buff bytes.Buffer
// So we can reset to the start of the reader
tr := io.TeeReader(r, &buff)
err := cr.prepare(tr)
if err != nil {
return nil, err
}
cr.c = csv.NewReader(&buff)
// The first one is a header, so let's just get rid of it
_, err = cr.c.Read()
if err != nil {
return nil, err
}
return []resource.Interface{resource.NewResourceDataset(do.Name, cr)}, nil
}
func (cr *reader) prepare(r io.Reader) (err error) {
cReader := csv.NewReader(r)
// Header
cr.header, err = cReader.Read()
if err != nil {
return err
}
// Entry count
for {
_, err := cReader.Read()
if err == io.EOF {
break
} else if err != nil {
return err
}
cr.count++
}
return nil
}
// Fields returns every available field in this dataset
func (cr *reader) Fields() []string {
return cr.header
}
// Next returns the field: value mapping for the next row
func (cr *reader) Next() (map[string]string, error) {
mr := make(map[string]string)
rr, err := cr.c.Read()
if err == io.EOF {
return nil, nil
} else if err != nil {
return nil, err
}
for i, h := range cr.header {
mr[h] = rr[i]
}
return mr, nil
}
+9
View File
@@ -0,0 +1,9 @@
package envoy
type (
// DecoderOpts provides additional context for source decoders
DecoderOpts struct {
Name string
Path string
}
)
+13 -5
View File
@@ -5,20 +5,22 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
)
type (
decoder interface {
CanDecodeFile(os.FileInfo) bool
Decode(context.Context, io.Reader) ([]resource.Interface, error)
Decode(context.Context, io.Reader, *envoy.DecoderOpts) ([]resource.Interface, error)
}
)
// DecodeDirectory is a helper to run the decoding process over the entire directory
func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.Interface, error) {
func Decode(ctx context.Context, p string, decoders ...decoder) ([]resource.Interface, error) {
var (
f *os.File
@@ -35,7 +37,7 @@ func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.I
return nil, fmt.Errorf("no decoders provided")
}
return nn, filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
return nn, filepath.Walk(p, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -56,12 +58,18 @@ func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.I
return nil
}
if f, err = os.Open(path); err != nil {
if f, err = os.Open(p); err != nil {
return err
}
defer f.Close()
if dnn, err = d.Decode(ctx, f); err != nil {
dir, fn := path.Split(p)
do := &envoy.DecoderOpts{
Name: fn,
Path: dir,
}
if dnn, err = d.Decode(ctx, f, do); err != nil {
return fmt.Errorf("failed to decode %s: %w", info.Name(), err)
}
+3 -3
View File
@@ -13,7 +13,7 @@ func TestGraph_Walk(t *testing.T) {
ctx := context.Background()
t.Run("simple node link; a -> b", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
a := &testResource{
resType: "test:resource:1:",
@@ -46,7 +46,7 @@ func TestGraph_Walk(t *testing.T) {
})
t.Run("cyclic node link; a -> b -> a", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
a := &testResource{
resType: "test:resource:1:",
@@ -83,7 +83,7 @@ func TestGraph_Walk(t *testing.T) {
})
t.Run("self-cycle; a -> a", func(t *testing.T) {
bl := NewGraphBuilder()
bl := NewBuilder()
a := &testResource{
resType: "test:resource:1:",
+3 -1
View File
@@ -2,6 +2,7 @@ package envoy
import (
"fmt"
"reflect"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
)
@@ -15,7 +16,8 @@ type (
// MarshalMerge takes one or more nodes and Marshals and merges all
func CollectNodes(ii ...interface{}) (nn []resource.Interface, err error) {
for _, i := range ii {
if i == nil {
// i == nil is not enough in go
if i == nil || (reflect.ValueOf(i).Kind() == reflect.Ptr && reflect.ValueOf(i).IsNil()) {
continue
}
+51 -1
View File
@@ -1,14 +1,25 @@
package resource
import (
"strings"
"github.com/cortezaproject/corteza-server/compose/types"
)
type (
rawSysValues struct {
OwnedBy string
CreatedAt string
CreatedBy string
UpdatedAt string
UpdatedBy string
DeletedAt string
DeletedBy string
}
ComposeRecordRaw struct {
ID string
Values map[string]string
SysValues map[string]string
SysValues *rawSysValues
RefUsers map[string]string
}
ComposeRecordRawSet []*ComposeRecordRaw
@@ -45,3 +56,42 @@ func NewComposeRecordSet(w crsWalker, nsRef, modRef string) *ComposeRecord {
return r
}
// ApplyValues takes in a raw map of things and creates a proper structure for it
func (cr *ComposeRecordRaw) ApplyValues(vv map[string]string) {
if cr.RefUsers == nil {
cr.RefUsers = make(map[string]string)
}
if cr.SysValues == nil {
cr.SysValues = &rawSysValues{}
}
if cr.Values == nil {
cr.Values = make(map[string]string)
}
for k, v := range vv {
switch strings.ToLower(k) {
case "ownedby":
cr.SysValues.OwnedBy = v
cr.RefUsers[v] = ""
case "createdat":
cr.SysValues.CreatedAt = v
case "createdby":
cr.SysValues.CreatedBy = v
cr.RefUsers[v] = ""
case "updatedat":
cr.SysValues.UpdatedAt = v
case "updatedby":
cr.SysValues.UpdatedBy = v
cr.RefUsers[v] = ""
case "deletedat":
cr.SysValues.DeletedAt = v
case "deletedby":
cr.SysValues.DeletedBy = v
cr.RefUsers[v] = ""
default:
cr.Values[k] = v
}
}
}
@@ -0,0 +1,93 @@
package resource
type (
// A simple wrapper struct for related things
composeRecordShaper struct{}
)
// ComposeRecordShaper initializes and returns a new compose record resource shaper
func ComposeRecordShaper() shaper {
return &composeRecordShaper{}
}
// Shape shapes ResourceDatasets based on the related compose template
//
// The first step finds the matching pair;
// The second step creates an actual resource based on the two.
func (crt *composeRecordShaper) Shape(rr []Interface) ([]Interface, error) {
// This will do for most cases
ii := make([]Interface, 0, int(len(rr)/2)+1)
for _, r := range rr {
rt, ok := r.(*ComposeRecordTemplate)
if !ok {
continue
}
rd := findResourceDataset(rr, rt.Identifiers())
if rd == nil {
return nil, genericSourceErrUnresolved(rt.Identifiers())
}
ii = append(ii, crt.toResource(rt, rd))
}
return ii, nil
}
func (crt *composeRecordShaper) toResource(def *ComposeRecordTemplate, dt *ResourceDataset) Interface {
w := func(f func(r *ComposeRecordRaw) error) error {
for {
mr, err := dt.P.Next()
if err != nil {
return err
}
if mr == nil {
return nil
}
// Get the bits in order
rRaw := &ComposeRecordRaw{
ID: crt.getKey(mr, def.Key),
}
rRaw.ApplyValues(crt.mapValues(mr, def.FieldMap))
// Process it
err = f(rRaw)
if err != nil {
return err
}
}
}
return NewComposeRecordSet(w, def.NsRef.Identifiers.First(), def.ModRef.Identifiers.First())
}
// mapValues maps original values based on the provided mapping
func (crt *composeRecordShaper) mapValues(ov map[string]string, fm MappingTplSet) map[string]string {
nv := make(map[string]string)
// Mappings are provided as a slice since we'll want to do some additional conditional mappings.
// We'll make an index for a nicer lookup for now.
mx := make(map[string]*MappingTpl)
for _, m := range fm {
mx[m.Cell] = m
}
for k, v := range ov {
if m, has := mx[k]; has {
nv[m.Field] = v
} else {
nv[k] = v
}
}
return nv
}
func (crt *composeRecordShaper) getKey(vv map[string]string, kk []string) (rtr string) {
for _, k := range kk {
rtr += k + "."
}
// Remove the trailing dot
return rtr[0 : len(rtr)-1]
}
@@ -0,0 +1,50 @@
package resource
type (
MappingTpl struct {
// Index is like Cell, but the index
Index uint
Cell string
Field string
// If specifies when the field should be mapped
// @todo
If string
// Expr specifies how to aditionally manipulate the value
// @todo
Expr string
}
MappingTplSet []*MappingTpl
ComposeRecordTemplate struct {
// We'll do this so it conforms to the resource.Interface
*base
ModRef *Ref
NsRef *Ref
// Name is the source name; topically file name
Name string
// Key determines what defines an identifier
Key []string
FieldMap MappingTplSet
}
)
// NewComposeRecordTemplate returns a record template based on the provided parameters.
//
// The template is only valid up until the resource shaping; it is not allowed after the fact.
func NewComposeRecordTemplate(modRef, nsRef, name string, fieldMap MappingTplSet, key ...string) *ComposeRecordTemplate {
r := &ComposeRecordTemplate{base: &base{}}
r.Name = name
r.Key = key
r.FieldMap = fieldMap
r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef)
r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef)
r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE)
r.AddIdentifier(identifiers(name)...)
return r
}
+24
View File
@@ -0,0 +1,24 @@
package resource
type (
provider interface {
Fields() []string
Next() (map[string]string, error)
}
ResourceDataset struct {
*base
P provider
}
)
func NewResourceDataset(name string, p provider) *ResourceDataset {
r := &ResourceDataset{base: &base{}}
r.P = p
r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE)
r.AddIdentifier(identifiers(name)...)
return r
}
+64
View File
@@ -0,0 +1,64 @@
package resource
import "fmt"
type (
shaper interface {
Shape([]Interface) ([]Interface, error)
}
)
// Shape shapes ResourceDatasets based on their correlated Template
//
// During the shaping step, no data is removed, nor added to the original resource slice.
// The shapers are ran in the same order they were provided.
func Shape(rr []Interface, ss ...shaper) ([]Interface, error) {
ii := make([]Interface, 0, int(len(rr)/2))
// Firstly handle any resource shaping
for _, s := range ss {
sdd, err := s.Shape(rr)
if err != nil {
return nil, err
}
ii = append(ii, sdd...)
}
// Cleanup the final shaped resource slice.
// @todo make this cleaner!!1!'
//
// After this point, all of the data should be ready for further processing.
for _, r := range rr {
if _, ok := r.(*ComposeRecordTemplate); ok {
continue
}
if _, ok := r.(*ResourceDataset); ok {
continue
}
ii = append(ii, r)
}
return ii, nil
}
// findResourceDataset finds and returns the first dataset that matches the
// provided identifiers.
func findResourceDataset(rr []Interface, ii Identifiers) *ResourceDataset {
for _, r := range rr {
genR, ok := r.(*ResourceDataset)
if !ok {
continue
}
if !genR.Identifiers().HasAny(ii) {
continue
}
return genR
}
return nil
}
func genericSourceErrUnresolved(ii Identifiers) error {
return fmt.Errorf("generic source unresolved %v", ii.StringSlice())
}
+17
View File
@@ -42,6 +42,7 @@ var (
ROLE_RESOURCE_TYPE = st.RoleRBACResource.String()
SETTINGS_RESOURCE_TYPE = "system:setting:"
USER_RESOURCE_TYPE = st.UserRBACResource.String()
DATA_SOURCE_RESOURCE_TYPE = "data:raw:"
)
func MakeIdentifiers(ss ...string) Identifiers {
@@ -86,6 +87,14 @@ func (ri Identifiers) StringSlice() []string {
return ss
}
func (ri Identifiers) First() string {
ss := ri.StringSlice()
if len(ss) <= 0 {
return ""
}
return ss[0]
}
func (ss RefSet) FilterByResourceType(rt string) RefSet {
rr := make(RefSet, 0, len(ss))
@@ -98,6 +107,14 @@ func (ss RefSet) FilterByResourceType(rt string) RefSet {
return rr
}
func (ss RefSet) FirstByResourceType(rt string) *Ref {
rr := ss.FilterByResourceType(rt)
if len(rt) <= 0 {
return nil
}
return rr[0]
}
func (rr InterfaceSet) Walk(f func(r Interface) error) (err error) {
for _, r := range rr {
err = f(r)
+95
View File
@@ -11,10 +11,24 @@ import (
)
type (
mappingTpl struct {
resource.MappingTpl `yaml:",inline"`
}
mappingTplSet []*mappingTpl
composeRecordTpl struct {
Source string `yaml:"from"`
Key []string
Mapping mappingTplSet
}
composeModule struct {
res *types.Module
refNamespace string
rbac rbacRuleSet
recTpl *composeRecordTpl
}
composeModuleSet []*composeModule
@@ -114,6 +128,13 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) {
wrap.res.Fields = aux.set()
return nil
case "records":
if isKind(v, yaml.MappingNode) {
return v.Decode(&wrap.recTpl)
} else {
return nodeErr(n, "records definition must be a map")
}
}
return nil
@@ -122,12 +143,86 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) {
func (wrap composeModule) MarshalEnvoy() ([]resource.Interface, error) {
rs := resource.NewComposeModule(wrap.res, wrap.refNamespace)
var crs *resource.ComposeRecordTemplate
if wrap.recTpl != nil {
s := wrap.recTpl
mtt := make(resource.MappingTplSet, 0, len(s.Mapping))
for _, m := range s.Mapping {
mtt = append(mtt, &m.MappingTpl)
}
crs = resource.NewComposeRecordTemplate(rs.Identifiers().First(), wrap.refNamespace, s.Source, mtt, s.Key...)
}
return envoy.CollectNodes(
rs,
crs,
wrap.rbac.bindResource(rs),
)
}
func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error {
return eachMap(n, func(k, v *yaml.Node) (err error) {
switch k.Value {
case "source", "origin", "from":
rt.Source = v.Value
case "key", "index", "pk":
if !isKind(v, yaml.SequenceNode) {
rt.Key = []string{v.Value}
} else {
rt.Key = make([]string, 0, 3)
eachSeq(v, func(y *yaml.Node) error {
rt.Key = append(rt.Key, v.Value)
return nil
})
}
case "mapping", "map":
rt.Mapping = make(mappingTplSet, 0, 20)
// When provided as a sequence node, map the fields based on the index.
// first cell is mapped to the first sequence value, second cell to the second, and so on.
// Omit cells with empty, /, or - value.
if isKind(v, yaml.SequenceNode) {
i := uint(0)
eachSeq(v, func(y *yaml.Node) error {
if v.Value == "" || v.Value == "/" || v.Value == "-" {
return nil
}
rt.Mapping = append(rt.Mapping, &mappingTpl{
MappingTpl: resource.MappingTpl{
Index: i,
Field: v.Value,
},
})
i++
return nil
})
} else if isKind(v, yaml.MappingNode) {
// When provided as a mapping node, it can be a simple cell: field map
// or a more complex underlying structure.
eachMap(v, func(k, v *yaml.Node) error {
m := &mappingTpl{}
if isKind(v, yaml.MappingNode) {
v.Decode(m)
} else {
m.Field = v.Value
}
m.Cell = k.Value
rt.Mapping = append(rt.Mapping, m)
return nil
})
}
}
return nil
})
}
func (set *composeModuleFieldSet) UnmarshalYAML(n *yaml.Node) error {
return each(n, func(k, v *yaml.Node) (err error) {
var (
+3 -4
View File
@@ -96,11 +96,10 @@ func (wset composeRecordSet) MarshalEnvoy() ([]resource.Interface, error) {
r := &resource.ComposeRecordRaw{
// @todo change this probably
ID: res.values["id"],
Values: res.values,
RefUsers: res.refUser,
SysValues: res.sysValues,
}
r.ApplyValues(res.values)
r.ApplyValues(res.sysValues)
recMap[res.refModule].rr = append(recMap[res.refModule].rr, r)
}
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"path/filepath"
"strings"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
"gopkg.in/yaml.v3"
)
@@ -38,7 +39,7 @@ func (y *decoder) CanDecodeFile(i os.FileInfo) bool {
return false
}
func (y *decoder) Decode(ctx context.Context, r io.Reader) ([]resource.Interface, error) {
func (y *decoder) Decode(ctx context.Context, r io.Reader, dctx *envoy.DecoderOpts) ([]resource.Interface, error) {
var (
doc = &Document{}
)
+2 -2
View File
@@ -57,7 +57,7 @@ func yd(ctx context.Context, suite, fname string) ([]resource.Interface, error)
defer f.Close()
d := yaml.Decoder()
return d.Decode(ctx, f)
return d.Decode(ctx, f, nil)
}
func dd(ctx context.Context, suite string) ([]resource.Interface, error) {
@@ -69,7 +69,7 @@ func dd(ctx context.Context, suite string) ([]resource.Interface, error) {
func encode(ctx context.Context, s store.Storer, nn []resource.Interface) error {
se := es.NewStoreEncoder(s, nil)
bld := envoy.NewGraphBuilder(se)
bld := envoy.NewBuilder(se)
g, err := bld.Build(ctx, nn...)
if err != nil {
return err