From 10b300328d660560601973cefb9610fc9e8c8ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Thu, 26 Nov 2020 18:39:50 +0100 Subject: [PATCH] Implement data shaping Unstructured datasources (csv, jsonl, ...) are defined as resource datasets. These datasets are then "shaped" based on some template. The result is an envoy resource like any other. --- pkg/envoy/builder.go | 8 +- pkg/envoy/builder_test.go | 10 +- pkg/envoy/csv/decoder.go | 105 ++++++++++++++++++ pkg/envoy/decoder.go | 9 ++ pkg/envoy/directory/decode.go | 18 ++- pkg/envoy/graph_test.go | 6 +- pkg/envoy/marshaler.go | 4 +- pkg/envoy/resource/compose_record.go | 52 ++++++++- pkg/envoy/resource/compose_record_shaper.go | 93 ++++++++++++++++ pkg/envoy/resource/compose_record_template.go | 50 +++++++++ pkg/envoy/resource/dataset.go | 24 ++++ pkg/envoy/resource/shaper.go | 64 +++++++++++ pkg/envoy/resource/types.go | 17 +++ pkg/envoy/yaml/compose_module.go | 95 ++++++++++++++++ pkg/envoy/yaml/compose_record.go | 7 +- pkg/envoy/yaml/decoder.go | 3 +- tests/envoy/main.go | 4 +- 17 files changed, 543 insertions(+), 26 deletions(-) create mode 100644 pkg/envoy/csv/decoder.go create mode 100644 pkg/envoy/decoder.go create mode 100644 pkg/envoy/resource/compose_record_shaper.go create mode 100644 pkg/envoy/resource/compose_record_template.go create mode 100644 pkg/envoy/resource/dataset.go create mode 100644 pkg/envoy/resource/shaper.go diff --git a/pkg/envoy/builder.go b/pkg/envoy/builder.go index 3a125aa65..e2dc57ec7 100644 --- a/pkg/envoy/builder.go +++ b/pkg/envoy/builder.go @@ -7,7 +7,7 @@ import ( ) type ( - graphBuilder struct { + builder struct { pp []Preparer } @@ -31,13 +31,13 @@ type ( } ) -func NewGraphBuilder(pp ...Preparer) *graphBuilder { - return &graphBuilder{ +func NewBuilder(pp ...Preparer) *builder { + return &builder{ pp: pp, } } -func (b *graphBuilder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) { +func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) { g := newGraph() // Prepare nodes for all resources diff --git a/pkg/envoy/builder_test.go b/pkg/envoy/builder_test.go index 5c76517ee..2297474d7 100644 --- a/pkg/envoy/builder_test.go +++ b/pkg/envoy/builder_test.go @@ -33,7 +33,7 @@ func TestGraphBuilder_Rel(t *testing.T) { ctx := context.Background() t.Run("single, simple node; a", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() rr := []resource.Interface{ &testResource{ @@ -54,7 +54,7 @@ func TestGraphBuilder_Rel(t *testing.T) { }) t.Run("simple node link; a -> b", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() rr := []resource.Interface{ &testResource{ @@ -84,7 +84,7 @@ func TestGraphBuilder_Rel(t *testing.T) { }) t.Run("cyclic node link; a -> b -> a", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() rr := []resource.Interface{ &testResource{ @@ -118,7 +118,7 @@ func TestGraphBuilder_Rel(t *testing.T) { }) t.Run("node with missing dep; a -> nill", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() rr := []resource.Interface{ &testResource{ @@ -139,7 +139,7 @@ func TestGraphBuilder_Rel(t *testing.T) { }) t.Run("self-cycle; a -> a", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() rr := []resource.Interface{ &testResource{ diff --git a/pkg/envoy/csv/decoder.go b/pkg/envoy/csv/decoder.go new file mode 100644 index 000000000..26383a8aa --- /dev/null +++ b/pkg/envoy/csv/decoder.go @@ -0,0 +1,105 @@ +package csv + +import ( + "bytes" + "context" + "encoding/csv" + "io" + "os" + "path/filepath" + "strings" + + "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" +) + +type ( + // wrapper struct for csv related methods + decoder struct{} + + // csv decoder wrapper for additional bits + reader struct { + c *csv.Reader + header []string + count uint64 + } +) + +// Decoder initializes and returns a fresh CSV decoder +func Decoder() *decoder { + return &decoder{} +} + +// CanDecodeFile determines if the file can be determined by this decoder +func (y *decoder) CanDecodeFile(i os.FileInfo) bool { + return strings.Trim(filepath.Ext(i.Name()), ".") == "csv" +} + +// Decode decodes the given io.Reader into a generic resource dataset +func (c *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts) ([]resource.Interface, error) { + cr := &reader{} + var buff bytes.Buffer + + // So we can reset to the start of the reader + tr := io.TeeReader(r, &buff) + err := cr.prepare(tr) + if err != nil { + return nil, err + } + + cr.c = csv.NewReader(&buff) + + // The first one is a header, so let's just get rid of it + _, err = cr.c.Read() + if err != nil { + return nil, err + } + + return []resource.Interface{resource.NewResourceDataset(do.Name, cr)}, nil +} + +func (cr *reader) prepare(r io.Reader) (err error) { + cReader := csv.NewReader(r) + + // Header + cr.header, err = cReader.Read() + if err != nil { + return err + } + + // Entry count + for { + _, err := cReader.Read() + if err == io.EOF { + break + } else if err != nil { + return err + } + + cr.count++ + } + + return nil +} + +// Fields returns every available field in this dataset +func (cr *reader) Fields() []string { + return cr.header +} + +// Next returns the field: value mapping for the next row +func (cr *reader) Next() (map[string]string, error) { + mr := make(map[string]string) + rr, err := cr.c.Read() + if err == io.EOF { + return nil, nil + } else if err != nil { + return nil, err + } + + for i, h := range cr.header { + mr[h] = rr[i] + } + + return mr, nil +} diff --git a/pkg/envoy/decoder.go b/pkg/envoy/decoder.go new file mode 100644 index 000000000..cc82e475c --- /dev/null +++ b/pkg/envoy/decoder.go @@ -0,0 +1,9 @@ +package envoy + +type ( + // DecoderOpts provides additional context for source decoders + DecoderOpts struct { + Name string + Path string + } +) diff --git a/pkg/envoy/directory/decode.go b/pkg/envoy/directory/decode.go index 24a699109..a668c141a 100644 --- a/pkg/envoy/directory/decode.go +++ b/pkg/envoy/directory/decode.go @@ -5,20 +5,22 @@ import ( "fmt" "io" "os" + "path" "path/filepath" + "github.com/cortezaproject/corteza-server/pkg/envoy" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" ) type ( decoder interface { CanDecodeFile(os.FileInfo) bool - Decode(context.Context, io.Reader) ([]resource.Interface, error) + Decode(context.Context, io.Reader, *envoy.DecoderOpts) ([]resource.Interface, error) } ) // DecodeDirectory is a helper to run the decoding process over the entire directory -func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.Interface, error) { +func Decode(ctx context.Context, p string, decoders ...decoder) ([]resource.Interface, error) { var ( f *os.File @@ -35,7 +37,7 @@ func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.I return nil, fmt.Errorf("no decoders provided") } - return nn, filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + return nn, filepath.Walk(p, func(p string, info os.FileInfo, err error) error { if err != nil { return err } @@ -56,12 +58,18 @@ func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.I return nil } - if f, err = os.Open(path); err != nil { + if f, err = os.Open(p); err != nil { return err } defer f.Close() - if dnn, err = d.Decode(ctx, f); err != nil { + dir, fn := path.Split(p) + do := &envoy.DecoderOpts{ + Name: fn, + Path: dir, + } + + if dnn, err = d.Decode(ctx, f, do); err != nil { return fmt.Errorf("failed to decode %s: %w", info.Name(), err) } diff --git a/pkg/envoy/graph_test.go b/pkg/envoy/graph_test.go index dc8ace0f8..50c1c1dd3 100644 --- a/pkg/envoy/graph_test.go +++ b/pkg/envoy/graph_test.go @@ -13,7 +13,7 @@ func TestGraph_Walk(t *testing.T) { ctx := context.Background() t.Run("simple node link; a -> b", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() a := &testResource{ resType: "test:resource:1:", @@ -46,7 +46,7 @@ func TestGraph_Walk(t *testing.T) { }) t.Run("cyclic node link; a -> b -> a", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() a := &testResource{ resType: "test:resource:1:", @@ -83,7 +83,7 @@ func TestGraph_Walk(t *testing.T) { }) t.Run("self-cycle; a -> a", func(t *testing.T) { - bl := NewGraphBuilder() + bl := NewBuilder() a := &testResource{ resType: "test:resource:1:", diff --git a/pkg/envoy/marshaler.go b/pkg/envoy/marshaler.go index 464cfc111..99c9db3f7 100644 --- a/pkg/envoy/marshaler.go +++ b/pkg/envoy/marshaler.go @@ -2,6 +2,7 @@ package envoy import ( "fmt" + "reflect" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" ) @@ -15,7 +16,8 @@ type ( // MarshalMerge takes one or more nodes and Marshals and merges all func CollectNodes(ii ...interface{}) (nn []resource.Interface, err error) { for _, i := range ii { - if i == nil { + // i == nil is not enough in go + if i == nil || (reflect.ValueOf(i).Kind() == reflect.Ptr && reflect.ValueOf(i).IsNil()) { continue } diff --git a/pkg/envoy/resource/compose_record.go b/pkg/envoy/resource/compose_record.go index 91f35f0d5..6b79f4be8 100644 --- a/pkg/envoy/resource/compose_record.go +++ b/pkg/envoy/resource/compose_record.go @@ -1,14 +1,25 @@ package resource import ( + "strings" + "github.com/cortezaproject/corteza-server/compose/types" ) type ( + rawSysValues struct { + OwnedBy string + CreatedAt string + CreatedBy string + UpdatedAt string + UpdatedBy string + DeletedAt string + DeletedBy string + } ComposeRecordRaw struct { ID string Values map[string]string - SysValues map[string]string + SysValues *rawSysValues RefUsers map[string]string } ComposeRecordRawSet []*ComposeRecordRaw @@ -45,3 +56,42 @@ func NewComposeRecordSet(w crsWalker, nsRef, modRef string) *ComposeRecord { return r } + +// ApplyValues takes in a raw map of things and creates a proper structure for it +func (cr *ComposeRecordRaw) ApplyValues(vv map[string]string) { + if cr.RefUsers == nil { + cr.RefUsers = make(map[string]string) + } + if cr.SysValues == nil { + cr.SysValues = &rawSysValues{} + } + if cr.Values == nil { + cr.Values = make(map[string]string) + } + + for k, v := range vv { + switch strings.ToLower(k) { + case "ownedby": + cr.SysValues.OwnedBy = v + cr.RefUsers[v] = "" + case "createdat": + cr.SysValues.CreatedAt = v + case "createdby": + cr.SysValues.CreatedBy = v + cr.RefUsers[v] = "" + case "updatedat": + cr.SysValues.UpdatedAt = v + case "updatedby": + cr.SysValues.UpdatedBy = v + cr.RefUsers[v] = "" + case "deletedat": + cr.SysValues.DeletedAt = v + case "deletedby": + cr.SysValues.DeletedBy = v + cr.RefUsers[v] = "" + + default: + cr.Values[k] = v + } + } +} diff --git a/pkg/envoy/resource/compose_record_shaper.go b/pkg/envoy/resource/compose_record_shaper.go new file mode 100644 index 000000000..e74488cdc --- /dev/null +++ b/pkg/envoy/resource/compose_record_shaper.go @@ -0,0 +1,93 @@ +package resource + +type ( + // A simple wrapper struct for related things + composeRecordShaper struct{} +) + +// ComposeRecordShaper initializes and returns a new compose record resource shaper +func ComposeRecordShaper() shaper { + return &composeRecordShaper{} +} + +// Shape shapes ResourceDatasets based on the related compose template +// +// The first step finds the matching pair; +// The second step creates an actual resource based on the two. +func (crt *composeRecordShaper) Shape(rr []Interface) ([]Interface, error) { + // This will do for most cases + ii := make([]Interface, 0, int(len(rr)/2)+1) + + for _, r := range rr { + rt, ok := r.(*ComposeRecordTemplate) + if !ok { + continue + } + rd := findResourceDataset(rr, rt.Identifiers()) + if rd == nil { + return nil, genericSourceErrUnresolved(rt.Identifiers()) + } + + ii = append(ii, crt.toResource(rt, rd)) + } + + return ii, nil +} + +func (crt *composeRecordShaper) toResource(def *ComposeRecordTemplate, dt *ResourceDataset) Interface { + w := func(f func(r *ComposeRecordRaw) error) error { + for { + mr, err := dt.P.Next() + if err != nil { + return err + } + if mr == nil { + return nil + } + + // Get the bits in order + rRaw := &ComposeRecordRaw{ + ID: crt.getKey(mr, def.Key), + } + rRaw.ApplyValues(crt.mapValues(mr, def.FieldMap)) + + // Process it + err = f(rRaw) + if err != nil { + return err + } + } + } + + return NewComposeRecordSet(w, def.NsRef.Identifiers.First(), def.ModRef.Identifiers.First()) +} + +// mapValues maps original values based on the provided mapping +func (crt *composeRecordShaper) mapValues(ov map[string]string, fm MappingTplSet) map[string]string { + nv := make(map[string]string) + + // Mappings are provided as a slice since we'll want to do some additional conditional mappings. + // We'll make an index for a nicer lookup for now. + mx := make(map[string]*MappingTpl) + for _, m := range fm { + mx[m.Cell] = m + } + + for k, v := range ov { + if m, has := mx[k]; has { + nv[m.Field] = v + } else { + nv[k] = v + } + } + + return nv +} + +func (crt *composeRecordShaper) getKey(vv map[string]string, kk []string) (rtr string) { + for _, k := range kk { + rtr += k + "." + } + // Remove the trailing dot + return rtr[0 : len(rtr)-1] +} diff --git a/pkg/envoy/resource/compose_record_template.go b/pkg/envoy/resource/compose_record_template.go new file mode 100644 index 000000000..59fa32061 --- /dev/null +++ b/pkg/envoy/resource/compose_record_template.go @@ -0,0 +1,50 @@ +package resource + +type ( + MappingTpl struct { + // Index is like Cell, but the index + Index uint + Cell string + Field string + + // If specifies when the field should be mapped + // @todo + If string + + // Expr specifies how to aditionally manipulate the value + // @todo + Expr string + } + MappingTplSet []*MappingTpl + + ComposeRecordTemplate struct { + // We'll do this so it conforms to the resource.Interface + *base + + ModRef *Ref + NsRef *Ref + + // Name is the source name; topically file name + Name string + // Key determines what defines an identifier + Key []string + FieldMap MappingTplSet + } +) + +// NewComposeRecordTemplate returns a record template based on the provided parameters. +// +// The template is only valid up until the resource shaping; it is not allowed after the fact. +func NewComposeRecordTemplate(modRef, nsRef, name string, fieldMap MappingTplSet, key ...string) *ComposeRecordTemplate { + r := &ComposeRecordTemplate{base: &base{}} + r.Name = name + r.Key = key + r.FieldMap = fieldMap + + r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef) + r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) + + r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE) + r.AddIdentifier(identifiers(name)...) + return r +} diff --git a/pkg/envoy/resource/dataset.go b/pkg/envoy/resource/dataset.go new file mode 100644 index 000000000..63777bf33 --- /dev/null +++ b/pkg/envoy/resource/dataset.go @@ -0,0 +1,24 @@ +package resource + +type ( + provider interface { + Fields() []string + Next() (map[string]string, error) + } + + ResourceDataset struct { + *base + + P provider + } +) + +func NewResourceDataset(name string, p provider) *ResourceDataset { + r := &ResourceDataset{base: &base{}} + r.P = p + + r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE) + r.AddIdentifier(identifiers(name)...) + + return r +} diff --git a/pkg/envoy/resource/shaper.go b/pkg/envoy/resource/shaper.go new file mode 100644 index 000000000..8910bb507 --- /dev/null +++ b/pkg/envoy/resource/shaper.go @@ -0,0 +1,64 @@ +package resource + +import "fmt" + +type ( + shaper interface { + Shape([]Interface) ([]Interface, error) + } +) + +// Shape shapes ResourceDatasets based on their correlated Template +// +// During the shaping step, no data is removed, nor added to the original resource slice. +// The shapers are ran in the same order they were provided. +func Shape(rr []Interface, ss ...shaper) ([]Interface, error) { + ii := make([]Interface, 0, int(len(rr)/2)) + + // Firstly handle any resource shaping + for _, s := range ss { + sdd, err := s.Shape(rr) + if err != nil { + return nil, err + } + ii = append(ii, sdd...) + } + + // Cleanup the final shaped resource slice. + // @todo make this cleaner!!1!' + // + // After this point, all of the data should be ready for further processing. + for _, r := range rr { + if _, ok := r.(*ComposeRecordTemplate); ok { + continue + } + if _, ok := r.(*ResourceDataset); ok { + continue + } + + ii = append(ii, r) + } + + return ii, nil +} + +// findResourceDataset finds and returns the first dataset that matches the +// provided identifiers. +func findResourceDataset(rr []Interface, ii Identifiers) *ResourceDataset { + for _, r := range rr { + genR, ok := r.(*ResourceDataset) + if !ok { + continue + } + if !genR.Identifiers().HasAny(ii) { + continue + } + + return genR + } + return nil +} + +func genericSourceErrUnresolved(ii Identifiers) error { + return fmt.Errorf("generic source unresolved %v", ii.StringSlice()) +} diff --git a/pkg/envoy/resource/types.go b/pkg/envoy/resource/types.go index ad83ac099..c62fdece0 100644 --- a/pkg/envoy/resource/types.go +++ b/pkg/envoy/resource/types.go @@ -42,6 +42,7 @@ var ( ROLE_RESOURCE_TYPE = st.RoleRBACResource.String() SETTINGS_RESOURCE_TYPE = "system:setting:" USER_RESOURCE_TYPE = st.UserRBACResource.String() + DATA_SOURCE_RESOURCE_TYPE = "data:raw:" ) func MakeIdentifiers(ss ...string) Identifiers { @@ -86,6 +87,14 @@ func (ri Identifiers) StringSlice() []string { return ss } +func (ri Identifiers) First() string { + ss := ri.StringSlice() + if len(ss) <= 0 { + return "" + } + return ss[0] +} + func (ss RefSet) FilterByResourceType(rt string) RefSet { rr := make(RefSet, 0, len(ss)) @@ -98,6 +107,14 @@ func (ss RefSet) FilterByResourceType(rt string) RefSet { return rr } +func (ss RefSet) FirstByResourceType(rt string) *Ref { + rr := ss.FilterByResourceType(rt) + if len(rt) <= 0 { + return nil + } + return rr[0] +} + func (rr InterfaceSet) Walk(f func(r Interface) error) (err error) { for _, r := range rr { err = f(r) diff --git a/pkg/envoy/yaml/compose_module.go b/pkg/envoy/yaml/compose_module.go index 6075b9d6d..c89d3f049 100644 --- a/pkg/envoy/yaml/compose_module.go +++ b/pkg/envoy/yaml/compose_module.go @@ -11,10 +11,24 @@ import ( ) type ( + mappingTpl struct { + resource.MappingTpl `yaml:",inline"` + } + mappingTplSet []*mappingTpl + + composeRecordTpl struct { + Source string `yaml:"from"` + + Key []string + Mapping mappingTplSet + } + composeModule struct { res *types.Module refNamespace string rbac rbacRuleSet + + recTpl *composeRecordTpl } composeModuleSet []*composeModule @@ -114,6 +128,13 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) { wrap.res.Fields = aux.set() return nil + case "records": + if isKind(v, yaml.MappingNode) { + return v.Decode(&wrap.recTpl) + } else { + return nodeErr(n, "records definition must be a map") + } + } return nil @@ -122,12 +143,86 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) { func (wrap composeModule) MarshalEnvoy() ([]resource.Interface, error) { rs := resource.NewComposeModule(wrap.res, wrap.refNamespace) + + var crs *resource.ComposeRecordTemplate + if wrap.recTpl != nil { + s := wrap.recTpl + + mtt := make(resource.MappingTplSet, 0, len(s.Mapping)) + for _, m := range s.Mapping { + mtt = append(mtt, &m.MappingTpl) + } + crs = resource.NewComposeRecordTemplate(rs.Identifiers().First(), wrap.refNamespace, s.Source, mtt, s.Key...) + } + return envoy.CollectNodes( rs, + crs, wrap.rbac.bindResource(rs), ) } +func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error { + return eachMap(n, func(k, v *yaml.Node) (err error) { + switch k.Value { + case "source", "origin", "from": + rt.Source = v.Value + + case "key", "index", "pk": + if !isKind(v, yaml.SequenceNode) { + rt.Key = []string{v.Value} + } else { + rt.Key = make([]string, 0, 3) + eachSeq(v, func(y *yaml.Node) error { + rt.Key = append(rt.Key, v.Value) + return nil + }) + } + + case "mapping", "map": + rt.Mapping = make(mappingTplSet, 0, 20) + // When provided as a sequence node, map the fields based on the index. + // first cell is mapped to the first sequence value, second cell to the second, and so on. + // Omit cells with empty, /, or - value. + if isKind(v, yaml.SequenceNode) { + i := uint(0) + eachSeq(v, func(y *yaml.Node) error { + if v.Value == "" || v.Value == "/" || v.Value == "-" { + return nil + } + + rt.Mapping = append(rt.Mapping, &mappingTpl{ + MappingTpl: resource.MappingTpl{ + Index: i, + Field: v.Value, + }, + }) + i++ + return nil + }) + } else if isKind(v, yaml.MappingNode) { + // When provided as a mapping node, it can be a simple cell: field map + // or a more complex underlying structure. + eachMap(v, func(k, v *yaml.Node) error { + m := &mappingTpl{} + + if isKind(v, yaml.MappingNode) { + v.Decode(m) + } else { + m.Field = v.Value + } + + m.Cell = k.Value + rt.Mapping = append(rt.Mapping, m) + return nil + }) + } + } + + return nil + }) +} + func (set *composeModuleFieldSet) UnmarshalYAML(n *yaml.Node) error { return each(n, func(k, v *yaml.Node) (err error) { var ( diff --git a/pkg/envoy/yaml/compose_record.go b/pkg/envoy/yaml/compose_record.go index 3666c6fc0..b5c563ba5 100644 --- a/pkg/envoy/yaml/compose_record.go +++ b/pkg/envoy/yaml/compose_record.go @@ -96,11 +96,10 @@ func (wset composeRecordSet) MarshalEnvoy() ([]resource.Interface, error) { r := &resource.ComposeRecordRaw{ // @todo change this probably ID: res.values["id"], - - Values: res.values, - RefUsers: res.refUser, - SysValues: res.sysValues, } + r.ApplyValues(res.values) + r.ApplyValues(res.sysValues) + recMap[res.refModule].rr = append(recMap[res.refModule].rr, r) } diff --git a/pkg/envoy/yaml/decoder.go b/pkg/envoy/yaml/decoder.go index e75a7c2ac..d2b7e1265 100644 --- a/pkg/envoy/yaml/decoder.go +++ b/pkg/envoy/yaml/decoder.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/cortezaproject/corteza-server/pkg/envoy" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "gopkg.in/yaml.v3" ) @@ -38,7 +39,7 @@ func (y *decoder) CanDecodeFile(i os.FileInfo) bool { return false } -func (y *decoder) Decode(ctx context.Context, r io.Reader) ([]resource.Interface, error) { +func (y *decoder) Decode(ctx context.Context, r io.Reader, dctx *envoy.DecoderOpts) ([]resource.Interface, error) { var ( doc = &Document{} ) diff --git a/tests/envoy/main.go b/tests/envoy/main.go index bd994bca4..728963b6b 100644 --- a/tests/envoy/main.go +++ b/tests/envoy/main.go @@ -57,7 +57,7 @@ func yd(ctx context.Context, suite, fname string) ([]resource.Interface, error) defer f.Close() d := yaml.Decoder() - return d.Decode(ctx, f) + return d.Decode(ctx, f, nil) } func dd(ctx context.Context, suite string) ([]resource.Interface, error) { @@ -69,7 +69,7 @@ func dd(ctx context.Context, suite string) ([]resource.Interface, error) { func encode(ctx context.Context, s store.Storer, nn []resource.Interface) error { se := es.NewStoreEncoder(s, nil) - bld := envoy.NewGraphBuilder(se) + bld := envoy.NewBuilder(se) g, err := bld.Build(ctx, nn...) if err != nil { return err