Implement data shaping
Unstructured datasources (csv, jsonl, ...) are defined as resource datasets. These datasets are then "shaped" based on some template. The result is an envoy resource like any other.
This commit is contained in:
@@ -7,7 +7,7 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
graphBuilder struct {
|
||||
builder struct {
|
||||
pp []Preparer
|
||||
}
|
||||
|
||||
@@ -31,13 +31,13 @@ type (
|
||||
}
|
||||
)
|
||||
|
||||
func NewGraphBuilder(pp ...Preparer) *graphBuilder {
|
||||
return &graphBuilder{
|
||||
func NewBuilder(pp ...Preparer) *builder {
|
||||
return &builder{
|
||||
pp: pp,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *graphBuilder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) {
|
||||
func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) {
|
||||
g := newGraph()
|
||||
|
||||
// Prepare nodes for all resources
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("single, simple node; a", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
rr := []resource.Interface{
|
||||
&testResource{
|
||||
@@ -54,7 +54,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("simple node link; a -> b", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
rr := []resource.Interface{
|
||||
&testResource{
|
||||
@@ -84,7 +84,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("cyclic node link; a -> b -> a", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
rr := []resource.Interface{
|
||||
&testResource{
|
||||
@@ -118,7 +118,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("node with missing dep; a -> nill", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
rr := []resource.Interface{
|
||||
&testResource{
|
||||
@@ -139,7 +139,7 @@ func TestGraphBuilder_Rel(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("self-cycle; a -> a", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
rr := []resource.Interface{
|
||||
&testResource{
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package csv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy"
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
|
||||
)
|
||||
|
||||
type (
|
||||
// wrapper struct for csv related methods
|
||||
decoder struct{}
|
||||
|
||||
// csv decoder wrapper for additional bits
|
||||
reader struct {
|
||||
c *csv.Reader
|
||||
header []string
|
||||
count uint64
|
||||
}
|
||||
)
|
||||
|
||||
// Decoder initializes and returns a fresh CSV decoder
|
||||
func Decoder() *decoder {
|
||||
return &decoder{}
|
||||
}
|
||||
|
||||
// CanDecodeFile determines if the file can be determined by this decoder
|
||||
func (y *decoder) CanDecodeFile(i os.FileInfo) bool {
|
||||
return strings.Trim(filepath.Ext(i.Name()), ".") == "csv"
|
||||
}
|
||||
|
||||
// Decode decodes the given io.Reader into a generic resource dataset
|
||||
func (c *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts) ([]resource.Interface, error) {
|
||||
cr := &reader{}
|
||||
var buff bytes.Buffer
|
||||
|
||||
// So we can reset to the start of the reader
|
||||
tr := io.TeeReader(r, &buff)
|
||||
err := cr.prepare(tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cr.c = csv.NewReader(&buff)
|
||||
|
||||
// The first one is a header, so let's just get rid of it
|
||||
_, err = cr.c.Read()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []resource.Interface{resource.NewResourceDataset(do.Name, cr)}, nil
|
||||
}
|
||||
|
||||
func (cr *reader) prepare(r io.Reader) (err error) {
|
||||
cReader := csv.NewReader(r)
|
||||
|
||||
// Header
|
||||
cr.header, err = cReader.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Entry count
|
||||
for {
|
||||
_, err := cReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cr.count++
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fields returns every available field in this dataset
|
||||
func (cr *reader) Fields() []string {
|
||||
return cr.header
|
||||
}
|
||||
|
||||
// Next returns the field: value mapping for the next row
|
||||
func (cr *reader) Next() (map[string]string, error) {
|
||||
mr := make(map[string]string)
|
||||
rr, err := cr.c.Read()
|
||||
if err == io.EOF {
|
||||
return nil, nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, h := range cr.header {
|
||||
mr[h] = rr[i]
|
||||
}
|
||||
|
||||
return mr, nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package envoy
|
||||
|
||||
type (
|
||||
// DecoderOpts provides additional context for source decoders
|
||||
DecoderOpts struct {
|
||||
Name string
|
||||
Path string
|
||||
}
|
||||
)
|
||||
@@ -5,20 +5,22 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy"
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
|
||||
)
|
||||
|
||||
type (
|
||||
decoder interface {
|
||||
CanDecodeFile(os.FileInfo) bool
|
||||
Decode(context.Context, io.Reader) ([]resource.Interface, error)
|
||||
Decode(context.Context, io.Reader, *envoy.DecoderOpts) ([]resource.Interface, error)
|
||||
}
|
||||
)
|
||||
|
||||
// DecodeDirectory is a helper to run the decoding process over the entire directory
|
||||
func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.Interface, error) {
|
||||
func Decode(ctx context.Context, p string, decoders ...decoder) ([]resource.Interface, error) {
|
||||
var (
|
||||
f *os.File
|
||||
|
||||
@@ -35,7 +37,7 @@ func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.I
|
||||
return nil, fmt.Errorf("no decoders provided")
|
||||
}
|
||||
|
||||
return nn, filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
||||
return nn, filepath.Walk(p, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -56,12 +58,18 @@ func Decode(ctx context.Context, path string, decoders ...decoder) ([]resource.I
|
||||
return nil
|
||||
}
|
||||
|
||||
if f, err = os.Open(path); err != nil {
|
||||
if f, err = os.Open(p); err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if dnn, err = d.Decode(ctx, f); err != nil {
|
||||
dir, fn := path.Split(p)
|
||||
do := &envoy.DecoderOpts{
|
||||
Name: fn,
|
||||
Path: dir,
|
||||
}
|
||||
|
||||
if dnn, err = d.Decode(ctx, f, do); err != nil {
|
||||
return fmt.Errorf("failed to decode %s: %w", info.Name(), err)
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ func TestGraph_Walk(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("simple node link; a -> b", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
a := &testResource{
|
||||
resType: "test:resource:1:",
|
||||
@@ -46,7 +46,7 @@ func TestGraph_Walk(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("cyclic node link; a -> b -> a", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
a := &testResource{
|
||||
resType: "test:resource:1:",
|
||||
@@ -83,7 +83,7 @@ func TestGraph_Walk(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("self-cycle; a -> a", func(t *testing.T) {
|
||||
bl := NewGraphBuilder()
|
||||
bl := NewBuilder()
|
||||
|
||||
a := &testResource{
|
||||
resType: "test:resource:1:",
|
||||
|
||||
@@ -2,6 +2,7 @@ package envoy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
|
||||
)
|
||||
@@ -15,7 +16,8 @@ type (
|
||||
// MarshalMerge takes one or more nodes and Marshals and merges all
|
||||
func CollectNodes(ii ...interface{}) (nn []resource.Interface, err error) {
|
||||
for _, i := range ii {
|
||||
if i == nil {
|
||||
// i == nil is not enough in go
|
||||
if i == nil || (reflect.ValueOf(i).Kind() == reflect.Ptr && reflect.ValueOf(i).IsNil()) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,25 @@
|
||||
package resource
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/compose/types"
|
||||
)
|
||||
|
||||
type (
|
||||
rawSysValues struct {
|
||||
OwnedBy string
|
||||
CreatedAt string
|
||||
CreatedBy string
|
||||
UpdatedAt string
|
||||
UpdatedBy string
|
||||
DeletedAt string
|
||||
DeletedBy string
|
||||
}
|
||||
ComposeRecordRaw struct {
|
||||
ID string
|
||||
Values map[string]string
|
||||
SysValues map[string]string
|
||||
SysValues *rawSysValues
|
||||
RefUsers map[string]string
|
||||
}
|
||||
ComposeRecordRawSet []*ComposeRecordRaw
|
||||
@@ -45,3 +56,42 @@ func NewComposeRecordSet(w crsWalker, nsRef, modRef string) *ComposeRecord {
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// ApplyValues takes in a raw map of things and creates a proper structure for it
|
||||
func (cr *ComposeRecordRaw) ApplyValues(vv map[string]string) {
|
||||
if cr.RefUsers == nil {
|
||||
cr.RefUsers = make(map[string]string)
|
||||
}
|
||||
if cr.SysValues == nil {
|
||||
cr.SysValues = &rawSysValues{}
|
||||
}
|
||||
if cr.Values == nil {
|
||||
cr.Values = make(map[string]string)
|
||||
}
|
||||
|
||||
for k, v := range vv {
|
||||
switch strings.ToLower(k) {
|
||||
case "ownedby":
|
||||
cr.SysValues.OwnedBy = v
|
||||
cr.RefUsers[v] = ""
|
||||
case "createdat":
|
||||
cr.SysValues.CreatedAt = v
|
||||
case "createdby":
|
||||
cr.SysValues.CreatedBy = v
|
||||
cr.RefUsers[v] = ""
|
||||
case "updatedat":
|
||||
cr.SysValues.UpdatedAt = v
|
||||
case "updatedby":
|
||||
cr.SysValues.UpdatedBy = v
|
||||
cr.RefUsers[v] = ""
|
||||
case "deletedat":
|
||||
cr.SysValues.DeletedAt = v
|
||||
case "deletedby":
|
||||
cr.SysValues.DeletedBy = v
|
||||
cr.RefUsers[v] = ""
|
||||
|
||||
default:
|
||||
cr.Values[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package resource
|
||||
|
||||
type (
|
||||
// A simple wrapper struct for related things
|
||||
composeRecordShaper struct{}
|
||||
)
|
||||
|
||||
// ComposeRecordShaper initializes and returns a new compose record resource shaper
|
||||
func ComposeRecordShaper() shaper {
|
||||
return &composeRecordShaper{}
|
||||
}
|
||||
|
||||
// Shape shapes ResourceDatasets based on the related compose template
|
||||
//
|
||||
// The first step finds the matching pair;
|
||||
// The second step creates an actual resource based on the two.
|
||||
func (crt *composeRecordShaper) Shape(rr []Interface) ([]Interface, error) {
|
||||
// This will do for most cases
|
||||
ii := make([]Interface, 0, int(len(rr)/2)+1)
|
||||
|
||||
for _, r := range rr {
|
||||
rt, ok := r.(*ComposeRecordTemplate)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rd := findResourceDataset(rr, rt.Identifiers())
|
||||
if rd == nil {
|
||||
return nil, genericSourceErrUnresolved(rt.Identifiers())
|
||||
}
|
||||
|
||||
ii = append(ii, crt.toResource(rt, rd))
|
||||
}
|
||||
|
||||
return ii, nil
|
||||
}
|
||||
|
||||
func (crt *composeRecordShaper) toResource(def *ComposeRecordTemplate, dt *ResourceDataset) Interface {
|
||||
w := func(f func(r *ComposeRecordRaw) error) error {
|
||||
for {
|
||||
mr, err := dt.P.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the bits in order
|
||||
rRaw := &ComposeRecordRaw{
|
||||
ID: crt.getKey(mr, def.Key),
|
||||
}
|
||||
rRaw.ApplyValues(crt.mapValues(mr, def.FieldMap))
|
||||
|
||||
// Process it
|
||||
err = f(rRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return NewComposeRecordSet(w, def.NsRef.Identifiers.First(), def.ModRef.Identifiers.First())
|
||||
}
|
||||
|
||||
// mapValues maps original values based on the provided mapping
|
||||
func (crt *composeRecordShaper) mapValues(ov map[string]string, fm MappingTplSet) map[string]string {
|
||||
nv := make(map[string]string)
|
||||
|
||||
// Mappings are provided as a slice since we'll want to do some additional conditional mappings.
|
||||
// We'll make an index for a nicer lookup for now.
|
||||
mx := make(map[string]*MappingTpl)
|
||||
for _, m := range fm {
|
||||
mx[m.Cell] = m
|
||||
}
|
||||
|
||||
for k, v := range ov {
|
||||
if m, has := mx[k]; has {
|
||||
nv[m.Field] = v
|
||||
} else {
|
||||
nv[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return nv
|
||||
}
|
||||
|
||||
func (crt *composeRecordShaper) getKey(vv map[string]string, kk []string) (rtr string) {
|
||||
for _, k := range kk {
|
||||
rtr += k + "."
|
||||
}
|
||||
// Remove the trailing dot
|
||||
return rtr[0 : len(rtr)-1]
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package resource
|
||||
|
||||
type (
|
||||
MappingTpl struct {
|
||||
// Index is like Cell, but the index
|
||||
Index uint
|
||||
Cell string
|
||||
Field string
|
||||
|
||||
// If specifies when the field should be mapped
|
||||
// @todo
|
||||
If string
|
||||
|
||||
// Expr specifies how to aditionally manipulate the value
|
||||
// @todo
|
||||
Expr string
|
||||
}
|
||||
MappingTplSet []*MappingTpl
|
||||
|
||||
ComposeRecordTemplate struct {
|
||||
// We'll do this so it conforms to the resource.Interface
|
||||
*base
|
||||
|
||||
ModRef *Ref
|
||||
NsRef *Ref
|
||||
|
||||
// Name is the source name; topically file name
|
||||
Name string
|
||||
// Key determines what defines an identifier
|
||||
Key []string
|
||||
FieldMap MappingTplSet
|
||||
}
|
||||
)
|
||||
|
||||
// NewComposeRecordTemplate returns a record template based on the provided parameters.
|
||||
//
|
||||
// The template is only valid up until the resource shaping; it is not allowed after the fact.
|
||||
func NewComposeRecordTemplate(modRef, nsRef, name string, fieldMap MappingTplSet, key ...string) *ComposeRecordTemplate {
|
||||
r := &ComposeRecordTemplate{base: &base{}}
|
||||
r.Name = name
|
||||
r.Key = key
|
||||
r.FieldMap = fieldMap
|
||||
|
||||
r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef)
|
||||
r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef)
|
||||
|
||||
r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE)
|
||||
r.AddIdentifier(identifiers(name)...)
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package resource
|
||||
|
||||
type (
|
||||
provider interface {
|
||||
Fields() []string
|
||||
Next() (map[string]string, error)
|
||||
}
|
||||
|
||||
ResourceDataset struct {
|
||||
*base
|
||||
|
||||
P provider
|
||||
}
|
||||
)
|
||||
|
||||
func NewResourceDataset(name string, p provider) *ResourceDataset {
|
||||
r := &ResourceDataset{base: &base{}}
|
||||
r.P = p
|
||||
|
||||
r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE)
|
||||
r.AddIdentifier(identifiers(name)...)
|
||||
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package resource
|
||||
|
||||
import "fmt"
|
||||
|
||||
type (
|
||||
shaper interface {
|
||||
Shape([]Interface) ([]Interface, error)
|
||||
}
|
||||
)
|
||||
|
||||
// Shape shapes ResourceDatasets based on their correlated Template
|
||||
//
|
||||
// During the shaping step, no data is removed, nor added to the original resource slice.
|
||||
// The shapers are ran in the same order they were provided.
|
||||
func Shape(rr []Interface, ss ...shaper) ([]Interface, error) {
|
||||
ii := make([]Interface, 0, int(len(rr)/2))
|
||||
|
||||
// Firstly handle any resource shaping
|
||||
for _, s := range ss {
|
||||
sdd, err := s.Shape(rr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ii = append(ii, sdd...)
|
||||
}
|
||||
|
||||
// Cleanup the final shaped resource slice.
|
||||
// @todo make this cleaner!!1!'
|
||||
//
|
||||
// After this point, all of the data should be ready for further processing.
|
||||
for _, r := range rr {
|
||||
if _, ok := r.(*ComposeRecordTemplate); ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := r.(*ResourceDataset); ok {
|
||||
continue
|
||||
}
|
||||
|
||||
ii = append(ii, r)
|
||||
}
|
||||
|
||||
return ii, nil
|
||||
}
|
||||
|
||||
// findResourceDataset finds and returns the first dataset that matches the
|
||||
// provided identifiers.
|
||||
func findResourceDataset(rr []Interface, ii Identifiers) *ResourceDataset {
|
||||
for _, r := range rr {
|
||||
genR, ok := r.(*ResourceDataset)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !genR.Identifiers().HasAny(ii) {
|
||||
continue
|
||||
}
|
||||
|
||||
return genR
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func genericSourceErrUnresolved(ii Identifiers) error {
|
||||
return fmt.Errorf("generic source unresolved %v", ii.StringSlice())
|
||||
}
|
||||
@@ -42,6 +42,7 @@ var (
|
||||
ROLE_RESOURCE_TYPE = st.RoleRBACResource.String()
|
||||
SETTINGS_RESOURCE_TYPE = "system:setting:"
|
||||
USER_RESOURCE_TYPE = st.UserRBACResource.String()
|
||||
DATA_SOURCE_RESOURCE_TYPE = "data:raw:"
|
||||
)
|
||||
|
||||
func MakeIdentifiers(ss ...string) Identifiers {
|
||||
@@ -86,6 +87,14 @@ func (ri Identifiers) StringSlice() []string {
|
||||
return ss
|
||||
}
|
||||
|
||||
func (ri Identifiers) First() string {
|
||||
ss := ri.StringSlice()
|
||||
if len(ss) <= 0 {
|
||||
return ""
|
||||
}
|
||||
return ss[0]
|
||||
}
|
||||
|
||||
func (ss RefSet) FilterByResourceType(rt string) RefSet {
|
||||
rr := make(RefSet, 0, len(ss))
|
||||
|
||||
@@ -98,6 +107,14 @@ func (ss RefSet) FilterByResourceType(rt string) RefSet {
|
||||
return rr
|
||||
}
|
||||
|
||||
func (ss RefSet) FirstByResourceType(rt string) *Ref {
|
||||
rr := ss.FilterByResourceType(rt)
|
||||
if len(rt) <= 0 {
|
||||
return nil
|
||||
}
|
||||
return rr[0]
|
||||
}
|
||||
|
||||
func (rr InterfaceSet) Walk(f func(r Interface) error) (err error) {
|
||||
for _, r := range rr {
|
||||
err = f(r)
|
||||
|
||||
@@ -11,10 +11,24 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
mappingTpl struct {
|
||||
resource.MappingTpl `yaml:",inline"`
|
||||
}
|
||||
mappingTplSet []*mappingTpl
|
||||
|
||||
composeRecordTpl struct {
|
||||
Source string `yaml:"from"`
|
||||
|
||||
Key []string
|
||||
Mapping mappingTplSet
|
||||
}
|
||||
|
||||
composeModule struct {
|
||||
res *types.Module
|
||||
refNamespace string
|
||||
rbac rbacRuleSet
|
||||
|
||||
recTpl *composeRecordTpl
|
||||
}
|
||||
composeModuleSet []*composeModule
|
||||
|
||||
@@ -114,6 +128,13 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) {
|
||||
wrap.res.Fields = aux.set()
|
||||
return nil
|
||||
|
||||
case "records":
|
||||
if isKind(v, yaml.MappingNode) {
|
||||
return v.Decode(&wrap.recTpl)
|
||||
} else {
|
||||
return nodeErr(n, "records definition must be a map")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -122,12 +143,86 @@ func (wrap *composeModule) UnmarshalYAML(n *yaml.Node) (err error) {
|
||||
|
||||
func (wrap composeModule) MarshalEnvoy() ([]resource.Interface, error) {
|
||||
rs := resource.NewComposeModule(wrap.res, wrap.refNamespace)
|
||||
|
||||
var crs *resource.ComposeRecordTemplate
|
||||
if wrap.recTpl != nil {
|
||||
s := wrap.recTpl
|
||||
|
||||
mtt := make(resource.MappingTplSet, 0, len(s.Mapping))
|
||||
for _, m := range s.Mapping {
|
||||
mtt = append(mtt, &m.MappingTpl)
|
||||
}
|
||||
crs = resource.NewComposeRecordTemplate(rs.Identifiers().First(), wrap.refNamespace, s.Source, mtt, s.Key...)
|
||||
}
|
||||
|
||||
return envoy.CollectNodes(
|
||||
rs,
|
||||
crs,
|
||||
wrap.rbac.bindResource(rs),
|
||||
)
|
||||
}
|
||||
|
||||
func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error {
|
||||
return eachMap(n, func(k, v *yaml.Node) (err error) {
|
||||
switch k.Value {
|
||||
case "source", "origin", "from":
|
||||
rt.Source = v.Value
|
||||
|
||||
case "key", "index", "pk":
|
||||
if !isKind(v, yaml.SequenceNode) {
|
||||
rt.Key = []string{v.Value}
|
||||
} else {
|
||||
rt.Key = make([]string, 0, 3)
|
||||
eachSeq(v, func(y *yaml.Node) error {
|
||||
rt.Key = append(rt.Key, v.Value)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
case "mapping", "map":
|
||||
rt.Mapping = make(mappingTplSet, 0, 20)
|
||||
// When provided as a sequence node, map the fields based on the index.
|
||||
// first cell is mapped to the first sequence value, second cell to the second, and so on.
|
||||
// Omit cells with empty, /, or - value.
|
||||
if isKind(v, yaml.SequenceNode) {
|
||||
i := uint(0)
|
||||
eachSeq(v, func(y *yaml.Node) error {
|
||||
if v.Value == "" || v.Value == "/" || v.Value == "-" {
|
||||
return nil
|
||||
}
|
||||
|
||||
rt.Mapping = append(rt.Mapping, &mappingTpl{
|
||||
MappingTpl: resource.MappingTpl{
|
||||
Index: i,
|
||||
Field: v.Value,
|
||||
},
|
||||
})
|
||||
i++
|
||||
return nil
|
||||
})
|
||||
} else if isKind(v, yaml.MappingNode) {
|
||||
// When provided as a mapping node, it can be a simple cell: field map
|
||||
// or a more complex underlying structure.
|
||||
eachMap(v, func(k, v *yaml.Node) error {
|
||||
m := &mappingTpl{}
|
||||
|
||||
if isKind(v, yaml.MappingNode) {
|
||||
v.Decode(m)
|
||||
} else {
|
||||
m.Field = v.Value
|
||||
}
|
||||
|
||||
m.Cell = k.Value
|
||||
rt.Mapping = append(rt.Mapping, m)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (set *composeModuleFieldSet) UnmarshalYAML(n *yaml.Node) error {
|
||||
return each(n, func(k, v *yaml.Node) (err error) {
|
||||
var (
|
||||
|
||||
@@ -96,11 +96,10 @@ func (wset composeRecordSet) MarshalEnvoy() ([]resource.Interface, error) {
|
||||
r := &resource.ComposeRecordRaw{
|
||||
// @todo change this probably
|
||||
ID: res.values["id"],
|
||||
|
||||
Values: res.values,
|
||||
RefUsers: res.refUser,
|
||||
SysValues: res.sysValues,
|
||||
}
|
||||
r.ApplyValues(res.values)
|
||||
r.ApplyValues(res.sysValues)
|
||||
|
||||
recMap[res.refModule].rr = append(recMap[res.refModule].rr, r)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy"
|
||||
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -38,7 +39,7 @@ func (y *decoder) CanDecodeFile(i os.FileInfo) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (y *decoder) Decode(ctx context.Context, r io.Reader) ([]resource.Interface, error) {
|
||||
func (y *decoder) Decode(ctx context.Context, r io.Reader, dctx *envoy.DecoderOpts) ([]resource.Interface, error) {
|
||||
var (
|
||||
doc = &Document{}
|
||||
)
|
||||
|
||||
+2
-2
@@ -57,7 +57,7 @@ func yd(ctx context.Context, suite, fname string) ([]resource.Interface, error)
|
||||
defer f.Close()
|
||||
|
||||
d := yaml.Decoder()
|
||||
return d.Decode(ctx, f)
|
||||
return d.Decode(ctx, f, nil)
|
||||
}
|
||||
|
||||
func dd(ctx context.Context, suite string) ([]resource.Interface, error) {
|
||||
@@ -69,7 +69,7 @@ func dd(ctx context.Context, suite string) ([]resource.Interface, error) {
|
||||
|
||||
func encode(ctx context.Context, s store.Storer, nn []resource.Interface) error {
|
||||
se := es.NewStoreEncoder(s, nil)
|
||||
bld := envoy.NewGraphBuilder(se)
|
||||
bld := envoy.NewBuilder(se)
|
||||
g, err := bld.Build(ctx, nn...)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user