DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Refactoring decode/encode structure and logic

Browse Source
This commit is contained in:
Denis Arh
2020-10-20 20:04:51 +02:00
committed by Tomaž Jerman
parent e30495c900
commit ee2183c36f
37 changed files with 1382 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
compose/types/chart.go
View File

@@ -26,19 +26,19 @@ type (
}
ChartConfig struct {
Reports []*ChartConfigReport `json:"reports,omitempty" yaml:",omitempty"`
ColorScheme string `json:"colorScheme,omitempty" yaml:",omitempty"`
Reports []*ChartConfigReport `json:"reports,omitempty"`
ColorScheme string `json:"colorScheme,omitempty"`
}
ChartConfigReport struct {
Filter string `json:"filter" yaml:",omitempty"`
ModuleID uint64 `json:"moduleID,string,omitempty" yaml:"moduleID,omitempty"`
Metrics []map[string]interface{} `json:"metrics,omitempty" yaml:",omitempty"`
Dimensions []map[string]interface{} `json:"dimensions,omitempty" yaml:",omitempty"`
YAxis map[string]interface{} `json:"yAxis,omitempty" yaml:",omitempty"`
Filter string `json:"filter"`
ModuleID uint64 `json:"moduleID,string,omitempty"`
Metrics []map[string]interface{} `json:"metrics,omitempty"`
Dimensions []map[string]interface{} `json:"dimensions,omitempty"`
YAxis map[string]interface{} `json:"yAxis,omitempty"`
Renderer struct {
Version string `json:"version,omitempty" yaml:",omitempty"`
} `json:"renderer,omitempty" yaml:",omitempty"`
Version string `json:"version,omitempty" `
} `json:"renderer,omitempty"`
}
ChartFilter struct {

12
compose/types/module.go
View File

@@ -10,19 +10,19 @@ import (
type (
Module struct {
ID uint64 `json:"moduleID,string" yaml:"-"`
ID uint64 `json:"moduleID,string"`
Handle string `json:"handle"`
Name string `json:"name"`
Meta types.JSONText `json:"meta" yaml:",omitempty"`
Fields ModuleFieldSet `json:"fields" yaml:"-"`
Meta types.JSONText `json:"meta"`
Fields ModuleFieldSet `json:"fields"`
Labels map[string]string `json:"labels,omitempty"`
NamespaceID uint64 `json:"namespaceID,string"`
CreatedAt time.Time `json:"createdAt,omitempty" yaml:",omitempty"`
UpdatedAt *time.Time `json:"updatedAt,omitempty" yaml:",omitempty"`
DeletedAt *time.Time `json:"deletedAt,omitempty" yaml:",omitempty"`
CreatedAt time.Time `json:"createdAt,omitempty"`
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
DeletedAt *time.Time `json:"deletedAt,omitempty"`
}
ModuleFilter struct {

4
compose/types/namespace.go
View File

@@ -48,8 +48,8 @@ type (
}
NamespaceMeta struct {
Subtitle string `json:"subtitle,omitempty" yaml:",omitempty"`
Description string `json:"description,omitempty" yaml:",omitempty"`
Subtitle string `json:"subtitle,omitempty"`
Description string `json:"description,omitempty"`
}
)

20
compose/types/page.go
View File

@@ -42,16 +42,16 @@ type (
PageBlocks []PageBlock
PageBlock struct {
Title string `json:"title,omitempty" yaml:",omitempty"`
Description string `json:"description,omitempty" yaml:",omitempty"`
Options map[string]interface{} `json:"options,omitempty" yaml:",omitempty"`
Style PageBlockStyle `json:"style,omitempty" yaml:",omitempty"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
Style PageBlockStyle `json:"style,omitempty"`
Kind string `json:"kind"`
XYWH [4]int `json:"xywh" yaml:"xywh,flow"` // x,y,w,h
XYWH [4]int `json:"xywh"` // x,y,w,h
}
PageBlockStyle struct {
Variants map[string]string `json:"variants,omitempty" yaml:",omitempty,flow"`
Variants map[string]string `json:"variants,omitempty"`
}
PageFilter struct {
@@ -127,10 +127,10 @@ func (b *PageBlock) UnmarshalJSON(data []byte) (err error) {
type internalPageBlock PageBlock
i := struct {
internalPageBlock
X int `json:"x,omitempty" yaml:"-"`
Y int `json:"y,omitempty" yaml:"-"`
Width int `json:"width,omitempty" yaml:"-"`
Height int `json:"height,omitempty" yaml:"-"`
X int `json:"x,omitempty"`
Y int `json:"y,omitempty"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
}{}
if err = json.Unmarshal(data, &i); err != nil {

2
compose/types/record.go
View File

@@ -36,7 +36,7 @@ type (
ID uint64 `json:"recordID,string"`
ModuleID uint64 `json:"moduleID,string"`
Values RecordValueSet `json:"values,omitempty" yaml:"-"`
Values RecordValueSet `json:"values,omitempty"`
Labels map[string]string `json:"labels,omitempty"`

6
pkg/envoy/types/compose_module_node.go → pkg/envoy/compose_module_node.go
View File

@@ -1,4 +1,4 @@
package types
package envoy
import (
"strconv"
@@ -9,10 +9,10 @@ import (
type (
// ComposeModuleNode represents a ComposeModule
ComposeModuleNode struct {
Mod *ComposeModule
Mod *types.Module
// Related namespace
Ns *ComposeNamespace
Ns *types.Namespace
}
)

7
pkg/envoy/types/compose_namespace_node.go → pkg/envoy/compose_namespace_node.go
View File

@@ -1,14 +1,13 @@
package types
package envoy
import (
"strconv"
"github.com/cortezaproject/corteza-server/compose/types"
"strconv"
)
type (
ComposeNamespaceNode struct {
Ns *ComposeNamespace
Ns *types.Namespace
}
)

28
pkg/envoy/types/compose_record_node.go → pkg/envoy/compose_record_node.go
View File

@@ -1,17 +1,18 @@
package types
package envoy
import (
"github.com/cortezaproject/corteza-server/compose/types"
"strconv"
)
type (
RecordIterator func(func(*ComposeRecord) error) error
RecordIterator func(func(record *types.Record) error) error
ComposeRecordNode struct {
Walk RecordIterator
// Metafields for relationship management
Mod *ComposeModule
Mod *types.Module
}
)
@@ -74,3 +75,24 @@ func (n *ComposeRecordNode) Update(mm ...Node) {
}
}
}
// A little helper to extract identifiers for a given module
func identifiersForModule(m *types.Module) NodeIdentifiers {
ii := make(NodeIdentifiers, 0)
if m == nil {
return ii
}
if m.Handle != "" {
ii = ii.Add(m.Handle)
}
if m.Name != "" {
ii = ii.Add(m.Name)
}
if m.ID > 0 {
ii = ii.Add(strconv.FormatUint(m.ID, 10))
}
return ii
}

31
pkg/envoy/decoder/csv.go → pkg/envoy/csv/decoder.go
View File

@@ -1,17 +1,26 @@
package decoder
package csv
import (
"context"
"encoding/csv"
"errors"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"io"
"regexp"
"github.com/cortezaproject/corteza-server/pkg/envoy/types"
)
type (
CsvDecoder struct{}
RecordIterator func(func(*types.Record) error) error
ComposeRecordNode struct {
Walk RecordIterator
// Metafields for relationship management
Mod *types.Module
}
)
var (
@@ -40,15 +49,15 @@ func (c *CsvDecoder) validateHeader(header []string) error {
return nil
}
func (c *CsvDecoder) Decode(ctx context.Context, r io.Reader, filename string) ([]types.Node, error) {
n := &types.ComposeRecordNode{}
func (c *CsvDecoder) Decode(ctx context.Context, r io.Reader, filename string) ([]envoy.Node, error) {
n := &envoy.ComposeRecordNode{}
// Determine base module for dependency resolution
// -4 is to remove .csv ext
//
// @todo tweak this a bit
modRes := filename[0 : len(filename)-4]
mod := &types.ComposeModule{}
mod := &types.Module{}
mod.Handle = modRes
mod.Name = modRes
n.Mod = mod
@@ -79,7 +88,7 @@ func (c *CsvDecoder) Decode(ctx context.Context, r io.Reader, filename string) (
// Iterator function for providing records to be imported.
// This doesn't do any validation; that should be handled by other layers.
n.Walk = func(f func(*types.ComposeRecord) error) error {
n.Walk = func(f func(*types.Record) error) error {
for {
record, err := cr.Read()
if err == io.EOF {
@@ -89,16 +98,16 @@ func (c *CsvDecoder) Decode(ctx context.Context, r io.Reader, filename string) (
return err
}
rvs := make(types.ComposeRecordValueSet, 0)
rvs := make(types.RecordValueSet, 0)
for i, h := range header {
v := &types.ComposeRecordValue{}
v := &types.RecordValue{}
v.Name = h
v.Value = record[i]
rvs = append(rvs, v)
}
rec := &types.ComposeRecord{}
rec := &types.Record{}
rec.Values = rvs
err = f(rec)
@@ -108,5 +117,5 @@ func (c *CsvDecoder) Decode(ctx context.Context, r io.Reader, filename string) (
}
}
return []types.Node{n}, nil
return []envoy.Node{n}, nil
}

57
pkg/envoy/decoder/directory.go
View File

@@ -1,57 +0,0 @@
package decoder
import (
"context"
"os"
"path/filepath"
"strings"
"github.com/cortezaproject/corteza-server/pkg/envoy/types"
)
// DecodeDirectory is a helper to run the decoding process over the entire directory
func DecodeDirectory(ctx context.Context, path string) (types.NodeSet, error) {
nn := make(types.NodeSet, 0, 100)
// Decoders
dy := NewYamlDecoder()
dc := NewCsvDecoder()
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
// Decode sources
fn := info.Name()
if strings.HasSuffix(fn, ".yaml") {
ynn, err := dy.Decode(ctx, f)
if err != nil {
return nil
}
nn = append(nn, ynn...)
} else if strings.HasSuffix(fn, ".csv") {
cnn, err := dc.Decode(ctx, f, fn)
if err != nil {
return nil
}
nn = append(nn, cnn...)
}
return nil
})
if err != nil {
return nil, err
}
return nn, nil
}

167
pkg/envoy/decoder/yaml.go
View File

@@ -1,167 +0,0 @@
package decoder
import (
"context"
"errors"
"io"
"strings"
"github.com/cortezaproject/corteza-server/pkg/envoy/types"
"gopkg.in/yaml.v3"
)
type (
// YamlDecoder is a wrapper struct for yaml related methods
YamlDecoder struct{}
// Document defines the supported yaml structure
Document struct {
Namespace string
Namespaces types.ComposeNamespaceSet
Modules types.ComposeModuleSet
Records map[string]types.ComposeRecordSet
}
)
var (
ErrorCannotResolveNamespace = errors.New("yaml decoder: cannot resolve namespace")
)
func NewYamlDecoder() *YamlDecoder {
return &YamlDecoder{}
}
func (y *YamlDecoder) unmarshalDocument(r io.Reader) (*Document, error) {
var c *Document
buf := new(strings.Builder)
_, err := io.Copy(buf, r)
if err != nil {
return nil, err
}
err = yaml.Unmarshal([]byte(buf.String()), &c)
if err != nil {
return nil, err
}
return c, nil
}
// convert converts the decoded document into a set of envoy nodes
func (y *YamlDecoder) convert(c *Document) ([]types.Node, error) {
nn := make([]types.Node, 0, 100)
// In case of namespaces...
if c.Namespaces != nil {
nodes, err := y.convertNamespaces(c.Namespaces)
if err != nil {
return nil, err
}
nn = append(nn, nodes...)
}
ns := &types.ComposeNamespace{}
if c.Namespace != "" {
// In case of a namespace to provide dependencies
ns.Slug = c.Namespace
ns.Name = c.Namespace
} else if len(nn) > 0 {
// Try to fall back to a namespace node
ns = ((nn[0]).(*types.ComposeNamespaceNode)).Ns
} else {
// No good; we can't link with a namespace.
// @note This should be checked when converting Compose resources only.
// Some resources don't belong to a namespace.
return nil, ErrorCannotResolveNamespace
}
// In case of modules...
if c.Modules != nil {
nodes, err := y.convertModules(c.Modules, ns)
if err != nil {
return nil, err
}
nn = append(nn, nodes...)
}
if c.Records != nil {
for modRef, rr := range c.Records {
// We can define a basic module representation as it will be updated later
// during validation/runtime
mod := &types.ComposeModule{}
mod.Handle = modRef
mod.Name = modRef
nodes, err := y.convertRecords(rr, mod)
if err != nil {
return nil, err
}
nn = append(nn, nodes...)
}
}
return nn, nil
}
func (y *YamlDecoder) convertNamespaces(nss types.ComposeNamespaceSet) ([]types.Node, error) {
nn := make([]types.Node, 0, 2)
for _, ns := range nss {
nn = append(nn, &types.ComposeNamespaceNode{Ns: ns})
// Nested modules
if ns.Modules != nil {
mm, err := y.convertModules(ns.Modules, ns)
if err != nil {
return nil, err
}
nn = append(nn, mm...)
}
// @todo nested RBAC
}
return nn, nil
}
func (y *YamlDecoder) convertModules(mm types.ComposeModuleSet, ns *types.ComposeNamespace) ([]types.Node, error) {
nn := make([]types.Node, 0)
for _, m := range mm {
nn = append(nn, &types.ComposeModuleNode{
Mod: m,
Ns: ns,
})
// @todo nested resources; should there be any?
}
return nn, nil
}
func (y *YamlDecoder) convertRecords(rr types.ComposeRecordSet, m *types.ComposeModule) ([]types.Node, error) {
// Iterator function for providing records to be imported.
// This doesn't do any validation; that should be handled by other layers.
f := func(f func(*types.ComposeRecord) error) error {
for _, r := range rr {
err := f(r)
if err != nil {
return err
}
}
return nil
}
return []types.Node{&types.ComposeRecordNode{Mod: m, Walk: f}}, nil
}
func (y *YamlDecoder) Decode(ctx context.Context, r io.Reader) ([]types.Node, error) {
d, err := y.unmarshalDocument(r)
if err != nil {
return nil, err
}
return y.convert(d)
}

70
pkg/envoy/directory/decode.go Normal file
View File

@@ -0,0 +1,70 @@
package directory
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"io"
"os"
"path/filepath"
)
type (
decoder interface {
CanDecodeFile(os.FileInfo) bool
Decode(context.Context, io.Reader, os.FileInfo) ([]envoy.Node, error)
}
)
// DecodeDirectory is a helper to run the decoding process over the entire directory
func Decode(ctx context.Context, path string, decoders ...decoder) ([]envoy.Node, error) {
var (
f *os.File
d decoder
// decoded nodes
dnn envoy.NodeSet
// agregated nodes
nn = make(envoy.NodeSet, 0, 100)
)
if len(decoders) == 0 {
return nil, fmt.Errorf("no decoders provided")
}
return nn, filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
for _, d = range decoders {
// find compatible decoder
if d.CanDecodeFile(info) {
break
}
}
if d == nil {
// no decoder found
return nil
}
if f, err = os.Open(path); err != nil {
return err
}
defer f.Close()
if dnn, err = d.Decode(ctx, f, info); err != nil {
return err
}
nn = append(nn, dnn...)
return nil
})
}

10
pkg/envoy/encoder/encoder.go → pkg/envoy/encoder.go
View File

@@ -1,18 +1,16 @@
package encoder
package envoy
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy/types"
)
type (
Encoder interface {
Encode(context.Context, ...types.Node) error
Encode(context.Context, ...Node) error
}
Provider interface {
Next(context.Context) (node types.Node, parentNodes []types.Node, childNodes []types.Node, err error)
Next(context.Context) (node Node, parentNodes []Node, childNodes []Node, err error)
}
)
@@ -34,7 +32,7 @@ func Encode(ctx context.Context, p Provider, ee ...Encoder) error {
}
// Upgradable nodes need to be processed based on their parent nodes
if un, is := node.(types.NodeUpdater); is {
if un, is := node.(NodeUpdater); is {
un.Update(pp...)
}

190
pkg/envoy/encoder/store.go
View File

@@ -1,190 +0,0 @@
package encoder
import (
"context"
"time"
"github.com/cortezaproject/corteza-server/compose/service/values"
compTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/types"
"github.com/cortezaproject/corteza-server/store"
)
type (
StoreEncoder struct {
s store.Storer
}
)
var (
rvSanitizer = values.Sanitizer()
rvValidator = values.Validator()
)
func NewStoreEncoder(s store.Storer) *StoreEncoder {
return &StoreEncoder{
s: s,
}
}
func (se *StoreEncoder) Encode(ctx context.Context, nn ...types.Node) error {
for _, n := range nn {
switch n.Resource() {
case compTypes.NamespaceRBACResource.String():
ns := n.(*types.ComposeNamespaceNode)
_, err := se.encodeNamespace(ctx, ns)
if err != nil {
return err
}
case compTypes.ModuleRBACResource.String():
mod := n.(*types.ComposeModuleNode)
_, err := se.encodeModule(ctx, mod)
if err != nil {
return err
}
case "compose:record:":
rec := n.(*types.ComposeRecordNode)
err := se.encodeRecord(ctx, rec)
if err != nil {
return err
}
}
}
return nil
}
func (se *StoreEncoder) encodeNamespace(ctx context.Context, n *types.ComposeNamespaceNode) (*types.ComposeNamespace, error) {
cns := n.Ns
err := store.Tx(ctx, se.s, func(ctx context.Context, s store.Storer) error {
cns.ID = nextID()
if cns.CreatedAt.IsZero() {
cns.CreatedAt = time.Now()
}
if err := store.CreateComposeNamespace(ctx, s, &cns.Namespace); err != nil {
return err
}
return nil
})
if err != nil {
return nil, err
}
return cns, nil
}
func (se *StoreEncoder) encodeModule(ctx context.Context, m *types.ComposeModuleNode) (*types.ComposeModule, error) {
cmod := m.Mod
cns := m.Ns
err := store.Tx(ctx, se.s, func(ctx context.Context, s store.Storer) error {
cmod.ID = nextID()
cmod.NamespaceID = cns.ID
if cmod.CreatedAt.IsZero() {
cmod.CreatedAt = time.Now()
}
// Store the module
mod := &cmod.Module
if err := store.CreateComposeModule(ctx, s, mod); err != nil {
return err
}
// Store module fields
for _, cf := range cmod.Fields {
cf.ID = nextID()
cf.ModuleID = cmod.ID
f := &cf.ModuleField
if err := store.CreateComposeModuleField(ctx, s, f); err != nil {
return err
}
// Update the original module fields so dependant resourcess can proceed without issues
mod.Fields = append(mod.Fields, f)
}
return nil
})
if err != nil {
return nil, err
}
return cmod, nil
}
func (se *StoreEncoder) encodeRecord(ctx context.Context, m *types.ComposeRecordNode) error {
cmod := m.Mod
err := store.Tx(ctx, se.s, func(ctx context.Context, s store.Storer) error {
return m.Walk(func(cr *types.ComposeRecord) error {
cr.ID = nextID()
cr.ModuleID = cmod.ID
cr.NamespaceID = cmod.NamespaceID
if cr.CreatedAt.IsZero() {
cr.CreatedAt = time.Now()
}
rec := &cr.Record
rec.Values = make(compTypes.RecordValueSet, 0, 100)
// Process record values
for _, crv := range cr.Values {
crv.RecordID = rec.ID
rec.Values = append(rec.Values, &crv.RecordValue)
}
rec.Values.SetUpdatedFlag(true)
rec.Values = se.setDefaultComposeRecordValues(cmod, rec.Values)
rec.Values = rvSanitizer.Run(&cmod.Module, rec.Values)
err := rvValidator.Run(ctx, s, &cmod.Module, rec)
if err != nil {
return err
}
if err := store.CreateComposeRecord(ctx, s, &m.Mod.Module, rec); err != nil {
return err
}
return nil
})
})
if err != nil {
return err
}
return nil
}
// @note this method is coppied over from the compose/service/record.
// Would it be better to unify the two methods?
func (se *StoreEncoder) setDefaultComposeRecordValues(m *types.ComposeModule, vv compTypes.RecordValueSet) (out compTypes.RecordValueSet) {
out = vv
for _, f := range m.Fields {
if f.DefaultValue == nil {
continue
}
for i, dv := range f.DefaultValue {
// Default values on field are (might be) without field name and place
if !out.Has(f.Name, uint(i)) {
out = append(out, &compTypes.RecordValue{
Name: f.Name,
Value: dv.Value,
Place: uint(i),
})
}
}
}
return
}

66
pkg/envoy/graph.go
View File

@@ -4,8 +4,6 @@ import (
"context"
"errors"
"math"
"github.com/cortezaproject/corteza-server/pkg/envoy/types"
)
type (
@@ -15,7 +13,7 @@ type (
// based on the node properties.
// Refer to the documentation for additional details.
Graph struct {
nodes []types.Node
nodes []Node
// Since it's calculated on the fly, this is all we need
invert bool
@@ -23,10 +21,10 @@ type (
// A cycle is interpreted as a dependency conflict (deadlock).
// It's up to the graph's discretion to determine what node in the cycle will be used.
// There is no guarantee that this list will be consistent across multiple runs.
conflicts types.NodeSet
conflicts NodeSet
// Nodes aren't immediately removed from the graph, so they are firstly marked as processed
processed types.NodeSet
processed NodeSet
}
)
@@ -40,9 +38,9 @@ var (
// making one yourself.
func NewGraph() *Graph {
return &Graph{
conflicts: make(types.NodeSet, 0, 100),
processed: make(types.NodeSet, 0, 100),
nodes: make([]types.Node, 0, 100),
conflicts: make(NodeSet, 0, 100),
processed: make(NodeSet, 0, 100),
nodes: make([]Node, 0, 100),
invert: false,
}
}
@@ -51,19 +49,19 @@ func NewGraph() *Graph {
//
// The method doesn't do any existence checks for duplicates.
// It simply pushes the provided nodes.
func (g *Graph) Add(nn ...types.Node) {
func (g *Graph) Add(nn ...Node) {
g.nodes = append(g.nodes, nn...)
}
// Remove removes the set of nodes nn from the graph h
//
// The nodes can only be removed if it doesn't have any unprocessed dependencies (child nodes)
func (g *Graph) Remove(nn ...types.Node) {
func (g *Graph) Remove(nn ...Node) {
if len(nn) <= 0 {
return
}
mm := make([]types.Node, 0, len(g.nodes))
mm := make([]Node, 0, len(g.nodes))
for _, m := range g.nodes {
for _, n := range nn {
if g.nodesMatch(m, n) && g.canRemove(n) {
@@ -78,8 +76,8 @@ func (g *Graph) Remove(nn ...types.Node) {
}
// FindNode returns all nodes that match the given resource and identifiers
func (g *Graph) FindNode(res string, identifiers ...string) []types.Node {
nn := make([]types.Node, 0, len(identifiers))
func (g *Graph) FindNode(res string, identifiers ...string) []Node {
nn := make([]Node, 0, len(identifiers))
for _, n := range g.nodes {
if n.Matches(res, identifiers...) {
nn = append(nn, n)
@@ -97,7 +95,7 @@ func (g *Graph) Invert() {
}
// Children provides node n child nodes **excluding** processed nodes
func (g *Graph) Children(n types.Node) []types.Node {
func (g *Graph) Children(n Node) []Node {
if !g.invert {
return g.removeProcessedNodes(g.children(n))
}
@@ -105,7 +103,7 @@ func (g *Graph) Children(n types.Node) []types.Node {
}
// ChildrenA provides node n child nodes **including** processed nodes
func (g *Graph) ChildrenA(n types.Node) []types.Node {
func (g *Graph) ChildrenA(n Node) []Node {
if !g.invert {
return g.children(n)
}
@@ -113,7 +111,7 @@ func (g *Graph) ChildrenA(n types.Node) []types.Node {
}
// Parents provides node n parent nodes **excluding** processed nodes
func (g *Graph) Parents(n types.Node) []types.Node {
func (g *Graph) Parents(n Node) []Node {
if !g.invert {
return g.removeProcessedNodes(g.parents(n))
}
@@ -121,7 +119,7 @@ func (g *Graph) Parents(n types.Node) []types.Node {
}
// ParentsA provides node n parent nodes **incliding** processed nodes
func (g *Graph) ParentsA(n types.Node) []types.Node {
func (g *Graph) ParentsA(n Node) []Node {
if !g.invert {
return g.parents(n)
}
@@ -129,9 +127,9 @@ func (g *Graph) ParentsA(n types.Node) []types.Node {
}
// ParentsAC provides node n parent nodes **including** processed nodes, **excluding** conflicting nodes
func (g *Graph) ParentsAC(n types.Node) []types.Node {
func (g *Graph) ParentsAC(n Node) []Node {
pp := g.Parents(n)
mm := make([]types.Node, 0, int(math.Max(float64(len(pp)-len(g.conflicts)), 1.0)))
mm := make([]Node, 0, int(math.Max(float64(len(pp)-len(g.conflicts)), 1.0)))
for _, p := range pp {
if !g.conflicts.Has(p) {
mm = append(mm, p)
@@ -149,8 +147,8 @@ func (g *Graph) Validate() error {
}
// Nodes returns all unprocessed nodes in the given graph g
func (g *Graph) Nodes() []types.Node {
nn := make([]types.Node, 0, len(g.nodes))
func (g *Graph) Nodes() []Node {
nn := make([]Node, 0, len(g.nodes))
for _, n := range g.nodes {
if !g.processed.Has(n) {
nn = append(nn, n)
@@ -166,7 +164,7 @@ func (g *Graph) Nodes() []types.Node {
// * If there is a node with no parent nodes; select that as the next node.
// * If there is no node with no parent nodes; determine a conflicting node.
// This returns the conflicting node n, it's parents, it's children and an ErrorDependencyConflict.
func (g *Graph) Next(ctx context.Context) (n types.Node, pp []types.Node, cc []types.Node, err error) {
func (g *Graph) Next(ctx context.Context) (n Node, pp []Node, cc []Node, err error) {
if len(g.Nodes()) <= 0 {
return nil, nil, nil, nil
}
@@ -211,14 +209,14 @@ func (g *Graph) Next(ctx context.Context) (n types.Node, pp []types.Node, cc []t
// Helper methods
// ------------------------------------------------------------------------
func (g *Graph) nodesMatch(n, m types.Node) bool {
func (g *Graph) nodesMatch(n, m Node) bool {
mRes := m.Resource()
mIdd := m.Identifiers()
return n.Matches(mRes, mIdd...)
}
func (g *Graph) canRemove(n types.Node) bool {
func (g *Graph) canRemove(n Node) bool {
if len(g.nodes) <= 1 {
return true
}
@@ -233,9 +231,9 @@ func (g *Graph) canRemove(n types.Node) bool {
return true
}
func (g *Graph) markProcessed(nn ...types.Node) {
func (g *Graph) markProcessed(nn ...Node) {
if g.processed == nil {
g.processed = make(types.NodeSet, 0, len(nn))
g.processed = make(NodeSet, 0, len(nn))
}
for _, n := range nn {
@@ -246,9 +244,9 @@ func (g *Graph) markProcessed(nn ...types.Node) {
}
// helper to mark the node as a conflicting node
func (g *Graph) markConflicting(n types.Node) {
func (g *Graph) markConflicting(n Node) {
if g.conflicts == nil {
g.conflicts = make(types.NodeSet, 0, 1)
g.conflicts = make(NodeSet, 0, 1)
}
if !g.conflicts.Has(n) {
@@ -256,8 +254,8 @@ func (g *Graph) markConflicting(n types.Node) {
}
}
func (g *Graph) removeProcessedNodes(nn []types.Node) []types.Node {
mm := make([]types.Node, 0, len(nn))
func (g *Graph) removeProcessedNodes(nn []Node) []Node {
mm := make([]Node, 0, len(nn))
for _, n := range nn {
if !g.processed.Has(n) {
mm = append(mm, n)
@@ -266,8 +264,8 @@ func (g *Graph) removeProcessedNodes(nn []types.Node) []types.Node {
return mm
}
func (g *Graph) children(n types.Node) []types.Node {
nn := make([]types.Node, 0)
func (g *Graph) children(n Node) []Node {
nn := make([]Node, 0)
// A simple find all nodes that n is in a relationship with will do the trick
for res, IDs := range n.Relations() {
nn = append(nn, g.FindNode(res, IDs...)...)
@@ -276,8 +274,8 @@ func (g *Graph) children(n types.Node) []types.Node {
return nn
}
func (g *Graph) parents(n types.Node) []types.Node {
nn := make([]types.Node, 0)
func (g *Graph) parents(n Node) []Node {
nn := make([]Node, 0)
// A more complex, find all nodes that have n in their relationship.
// @note can we make this nicer?

2
pkg/envoy/types/node.go → pkg/envoy/node.go
View File

@@ -1,4 +1,4 @@
package types
package envoy
type (
// Node defines the signature of any valid graph node

165
pkg/envoy/store/encoder.go Normal file
View File

@@ -0,0 +1,165 @@
package store
import (
"context"
"github.com/cortezaproject/corteza-server/compose/service/values"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/store"
"time"
)
type (
StoreEncoder struct {
s store.Storer
}
)
var (
rvSanitizer = values.Sanitizer()
rvValidator = values.Validator()
)
func NewStoreEncoder(s store.Storer) *StoreEncoder {
return &StoreEncoder{
s: s,
}
}
func (se *StoreEncoder) Encode(ctx context.Context, s store.Storer, nn ...envoy.Node) error {
return store.Tx(ctx, se.s, func(ctx context.Context, s store.Storer) error {
for _, n := range nn {
switch n.Resource() {
case types.NamespaceRBACResource.String():
ns := n.(*envoy.ComposeNamespaceNode)
_, err := se.encodeNamespace(ctx, s, ns)
if err != nil {
return err
}
case types.ModuleRBACResource.String():
mod := n.(*envoy.ComposeModuleNode)
_, err := se.encodeModule(ctx, s, mod)
if err != nil {
return err
}
case "compose:record:":
rec := n.(*envoy.ComposeRecordNode)
err := se.encodeRecord(ctx, s, rec)
if err != nil {
return err
}
}
}
return nil
})
}
func (se *StoreEncoder) encodeNamespace(ctx context.Context, s store.Storer, n *envoy.ComposeNamespaceNode) (*types.Namespace, error) {
cns := n.Ns
cns.ID = nextID()
if cns.CreatedAt.IsZero() {
cns.CreatedAt = time.Now()
}
if err := store.CreateComposeNamespace(ctx, s, cns); err != nil {
return nil, err
}
return cns, nil
}
func (se *StoreEncoder) encodeModule(ctx context.Context, s store.Storer, m *envoy.ComposeModuleNode) (*types.Module, error) {
mod := m.Mod
cns := m.Ns
mod.ID = nextID()
mod.NamespaceID = cns.ID
if mod.CreatedAt.IsZero() {
mod.CreatedAt = time.Now()
}
// Store the module
if err := store.CreateComposeModule(ctx, s, mod); err != nil {
return nil, err
}
// Store module fields
for _, f := range mod.Fields {
f.ID = nextID()
f.ModuleID = mod.ID
if err := store.CreateComposeModuleField(ctx, s, f); err != nil {
return nil, err
}
// Update the original module fields so dependant resources can proceed without issues
mod.Fields = append(mod.Fields, f)
}
return mod, nil
}
func (se *StoreEncoder) encodeRecord(ctx context.Context, s store.Storer, m *envoy.ComposeRecordNode) error {
mod := m.Mod
return m.Walk(func(rec *types.Record) error {
rec.ID = nextID()
rec.ModuleID = mod.ID
rec.NamespaceID = mod.NamespaceID
if rec.CreatedAt.IsZero() {
rec.CreatedAt = time.Now()
}
rec.Values = make(types.RecordValueSet, 0, 100)
// Process record values
for _, crv := range rec.Values {
crv.RecordID = rec.ID
rec.Values = append(rec.Values, crv)
}
rec.Values.SetUpdatedFlag(true)
rec.Values = se.setDefaultComposeRecordValues(mod, rec.Values)
rec.Values = rvSanitizer.Run(mod, rec.Values)
err := rvValidator.Run(ctx, s, mod, rec)
if err != nil {
return err
}
if err := store.CreateComposeRecord(ctx, s, m.Mod, rec); err != nil {
return err
}
return nil
})
}
// @note this method is coppied over from the compose/service/record.
// Would it be better to unify the two methods?
func (se *StoreEncoder) setDefaultComposeRecordValues(m *types.Module, vv types.RecordValueSet) (out types.RecordValueSet) {
out = vv
for _, f := range m.Fields {
if f.DefaultValue == nil {
continue
}
for i, dv := range f.DefaultValue {
// Default values on field are (might be) without field name and place
if !out.Has(f.Name, uint(i)) {
out = append(out, &types.RecordValue{
Name: f.Name,
Value: dv.Value,
Place: uint(i),
})
}
}
}
return
}

2
pkg/envoy/encoder/util.go → pkg/envoy/store/util.go
View File

@@ -1,4 +1,4 @@
package encoder
package store
import "github.com/cortezaproject/corteza-server/pkg/id"

86
pkg/envoy/types/compose_module.go
View File

@@ -1,86 +0,0 @@
package types
import (
compTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/util"
"gopkg.in/yaml.v3"
)
type (
ComposeModuleField struct {
compTypes.ModuleField `yaml:",inline"`
}
ComposeModuleFieldSet []*ComposeModuleField
ComposeModule struct {
compTypes.Module `yaml:",inline"`
Fields ComposeModuleFieldSet `yaml:"fields"`
Rbac `yaml:",inline"`
}
ComposeModuleSet []*ComposeModule
)
func (mm *ComposeModuleSet) UnmarshalYAML(n *yaml.Node) error {
cms := ComposeModuleSet{}
err := util.YamlIterator(n, func(n, m *yaml.Node) error {
handle := ""
if n != nil {
handle = n.Value
}
mod := &ComposeModule{}
err := m.Decode(mod)
if err != nil {
return err
}
if mod.Handle == "" {
mod.Handle = handle
}
if mod.Name == "" {
mod.Name = handle
}
cms = append(cms, mod)
return nil
})
if err != nil {
return err
}
*mm = cms
return nil
}
func (ff *ComposeModuleFieldSet) UnmarshalYAML(n *yaml.Node) error {
ffs := ComposeModuleFieldSet{}
err := util.YamlIterator(n, func(n, m *yaml.Node) error {
name := ""
if n != nil {
name = n.Value
}
f := &ComposeModuleField{}
err := m.Decode(f)
if err != nil {
return err
}
if f.Name == "" {
f.Name = name
}
if f.Label == "" {
f.Label = name
}
ffs = append(ffs, f)
return nil
})
if err != nil {
return err
}
*ff = ffs
return nil
}

48
pkg/envoy/types/compose_namespace.go
View File

@@ -1,48 +0,0 @@
package types
import (
compTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/util"
"gopkg.in/yaml.v3"
)
type (
ComposeNamespace struct {
compTypes.Namespace `yaml:",inline"`
Modules ComposeModuleSet
Rbac `yaml:",inline"`
}
ComposeNamespaceSet []*ComposeNamespace
)
func (ss *ComposeNamespaceSet) UnmarshalYAML(n *yaml.Node) error {
nss := ComposeNamespaceSet{}
err := util.YamlIterator(n, func(n, m *yaml.Node) error {
slug := ""
if n != nil {
slug = n.Value
}
ns := &ComposeNamespace{}
err := m.Decode(ns)
if err != nil {
return err
}
if ns.Slug == "" {
ns.Slug = slug
}
if ns.Name == "" {
ns.Name = slug
}
nss = append(nss, ns)
return nil
})
if err != nil {
return err
}
*ss = nss
return nil
}

72
pkg/envoy/types/compose_record.go
View File

@@ -1,72 +0,0 @@
package types
import (
"errors"
compTypes "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy/util"
"gopkg.in/yaml.v3"
)
type (
ComposeRecordValue struct {
compTypes.RecordValue `yaml:",inline"`
}
ComposeRecordValueSet []*ComposeRecordValue
ComposeRecord struct {
compTypes.Record `yaml:",inline"`
Values ComposeRecordValueSet `yaml:"values"`
}
ComposeRecordSet []*ComposeRecord
)
var (
ErrRecordDecoderMapNotSupported = errors.New("record decoder: maps not supported")
)
func (rr *ComposeRecordSet) UnmarshalYAML(n *yaml.Node) error {
crs := ComposeRecordSet{}
if n.Kind != yaml.SequenceNode {
return ErrRecordDecoderMapNotSupported
}
err := util.YamlIterator(n, func(n, m *yaml.Node) error {
var r *ComposeRecord
err := m.Decode(&r)
if err != nil {
return err
}
crs = append(crs, r)
return nil
})
if err != nil {
return err
}
*rr = crs
return nil
}
func (vv *ComposeRecordValueSet) UnmarshalYAML(n *yaml.Node) error {
vvs := ComposeRecordValueSet{}
err := util.YamlIterator(n, func(n, m *yaml.Node) error {
v := &ComposeRecordValue{}
v.Name = n.Value
v.Value = m.Value
v.Updated = true
vvs = append(vvs, v)
return nil
})
if err != nil {
return err
}
*vv = vvs
return nil
}

24
pkg/envoy/types/util.go
View File

@@ -1,24 +0,0 @@
package types
import "strconv"
// A little helper to extract identifiers for a given module
func identifiersForModule(m *ComposeModule) NodeIdentifiers {
ii := make(NodeIdentifiers, 0)
if m == nil {
return ii
}
if m.Handle != "" {
ii = ii.Add(m.Handle)
}
if m.Name != "" {
ii = ii.Add(m.Name)
}
if m.ID > 0 {
ii = ii.Add(strconv.FormatUint(m.ID, 10))
}
return ii
}

39
pkg/envoy/util/yaml.go
View File

@@ -1,39 +0,0 @@
package util
import (
"fmt"
"gopkg.in/yaml.v3"
)
func yamlNodeErr(n *yaml.Node, format string, aa ...interface{}) error {
format += " (%d:%d)"
aa = append(aa, n.Line, n.Column)
return fmt.Errorf(format, aa...)
}
// YamlIterator helps iterate over mapping and sequence nodes fairly trivially
func YamlIterator(n *yaml.Node, fn func(*yaml.Node, *yaml.Node) error) error {
if n.Kind == yaml.MappingNode {
for i := 0; i < len(n.Content); i += 2 {
if err := fn(n.Content[i], n.Content[i+1]); err != nil {
return err
}
}
return nil
}
if n.Kind == yaml.SequenceNode {
var placeholder *yaml.Node
for i := 0; i < len(n.Content); i++ {
if err := fn(placeholder, n.Content[i]); err != nil {
return err
}
}
return nil
}
return yamlNodeErr(n, "expecting mapping or sequence node")
}

38
pkg/envoy/yaml/compose.go Normal file
View File

@@ -0,0 +1,38 @@
package yaml
import (
"gopkg.in/yaml.v3"
)
type (
compose struct {
namespaces ComposeNamespaceSet
modules ComposeModuleSet
records ComposeRecordSet
// pages ComposePagesSet
// charts ComposeChartsSet
}
)
func (c *compose) UnmarshalYAML(n *yaml.Node) error {
if !isKind(n, yaml.MappingNode) {
// root node kind be mapping
return nodeErr(n, "expecting mapping node")
}
return iterator(n, func(k, v *yaml.Node) error {
switch k.Value {
case "namespace":
c.namespaces = ComposeNamespaceSet{&ComposeNamespace{}}
return v.Decode(&c.namespaces[0])
case "namespaces":
return v.Decode(&c.namespaces)
case "modules":
return v.Decode(&c.modules)
case "records":
return v.Decode(&c.records)
}
return nil
})
}

176
pkg/envoy/yaml/compose_module.go Normal file
View File

@@ -0,0 +1,176 @@
package yaml
import (
"fmt"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/handle"
"gopkg.in/yaml.v3"
)
type (
ComposeModule struct {
res *types.Module
Rbac `yaml:",inline"`
}
ComposeModuleSet []*ComposeModule
ComposeModuleField struct {
res *types.ModuleField `yaml:",inline"`
Rbac `yaml:",inline"`
}
ComposeModuleFieldSet []*ComposeModuleField
)
func (set *ComposeModuleSet) UnmarshalYAML(n *yaml.Node) error {
return iterator(n, func(k, v *yaml.Node) (err error) {
var (
wrap = &ComposeModule{}
)
if v == nil {
return nodeErr(n, "malformed module definition")
}
if err = v.Decode(&wrap); err != nil {
return
}
if k != nil {
if wrap.res.Handle != "" {
return nodeErr(k, "cannot define handle in mapped module definition")
}
if !handle.IsValid(k.Value) {
return nodeErr(n, "module reference must be a valid handle")
}
wrap.res.Handle = k.Value
wrap.res.Name = k.Value
}
*set = append(*set, wrap)
return
})
}
func (wrap *ComposeModule) UnmarshalYAML(n *yaml.Node) error {
if wrap.res == nil {
wrap.res = &types.Module{}
}
if !isKind(n, yaml.MappingNode) {
return nodeErr(n, "module definition must be a map")
}
return iterator(n, func(k, v *yaml.Node) (err error) {
switch k.Value {
case "name":
return decodeScalar(v, &wrap.res.Name)
case "fields":
if !isKind(v, yaml.MappingNode) {
return nodeErr(n, "field definition must be a map")
}
var (
aux = ComposeModuleFieldSet{}
)
if err = v.Decode(&aux); err != nil {
return err
}
wrap.res.Fields = aux.set()
return nil
}
return nil
})
}
func (set *ComposeModuleFieldSet) UnmarshalYAML(n *yaml.Node) error {
return iterator(n, func(k, v *yaml.Node) (err error) {
var (
wrap = &ComposeModuleField{}
)
if v == nil {
return nodeErr(n, "malformed module field definition")
}
if err = v.Decode(&wrap); err != nil {
return err
}
if k != nil {
if !handle.IsValid(k.Value) {
return nodeErr(n, "field name must be a valid handle")
}
wrap.res.Name = k.Value
wrap.res.Label = k.Value
}
*set = append(*set, wrap)
return
})
}
func (set ComposeModuleFieldSet) set() (out types.ModuleFieldSet) {
for _, i := range set {
out = append(out, i.res)
}
return out
}
func (wrap *ComposeModuleField) UnmarshalYAML(n *yaml.Node) error {
if wrap.res == nil {
wrap.res = &types.ModuleField{}
}
if !isKind(n, yaml.MappingNode) {
return nodeErr(n, "module field definition must be a map")
}
return iterator(n, func(k, v *yaml.Node) (err error) {
switch k.Value {
case "name":
return fmt.Errorf("name should be encoded as field definition key")
case "label":
return decodeScalar(v, &wrap.res.Label)
case "kind", "type":
return decodeScalar(v, &wrap.res.Kind)
case "options":
return v.Decode(&wrap.res.Options)
case "private":
return decodeScalar(v, &wrap.res.Private)
case "required":
return decodeScalar(v, &wrap.res.Required)
case "visible":
return decodeScalar(v, &wrap.res.Visible)
case "multi":
return decodeScalar(v, &wrap.res.Multi)
case "default":
return fmt.Errorf("field.default /// to be imple,emted")
//wrap.res.DefaultValue = types.RecordValueSet{}
//return deinterfacer.Each(val, func(place int, _ string, val interface{}) (err error) {
// field.DefaultValue = append(field.DefaultValue, &types.RecordValue{
// Value: deinterfacer.ToString(val),
// Place: uint(place),
// })
//
// return
//})
}
return nil
})
}

31
pkg/envoy/yaml/compose_module_test.go Normal file
View File

@@ -0,0 +1,31 @@
package yaml
import (
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"testing"
)
func TestComposeModule_UnmarshalYAML(t *testing.T) {
var (
req = require.New(t)
w *ComposeModule
err error
parse = func(w *ComposeModule, src string) (*ComposeModule, error) {
return w, yaml.Unmarshal([]byte(src), w)
}
)
w, err = parse(&ComposeModule{}, ``)
req.NoError(err)
req.NotNil(w)
req.Nil(w.res)
w, err = parse(&ComposeModule{}, `{ name: Test }`)
req.NoError(err)
req.NotNil(w)
req.NotNil(w.res)
req.NotEmpty(w.res.Name)
}

88
pkg/envoy/yaml/compose_namespace.go Normal file
View File

@@ -0,0 +1,88 @@
package yaml
import (
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/handle"
"gopkg.in/yaml.v3"
)
type (
ComposeNamespace struct {
res *types.Namespace `yaml:",inline"`
ref string
Modules ComposeModuleSet
Rbac `yaml:",inline"`
}
ComposeNamespaceSet []*ComposeNamespace
)
// UnmarshalYAML resolves set of namespace definitions, either sequence or map
//
// When resolving map, key is used as slug
//
//
func (wset *ComposeNamespaceSet) UnmarshalYAML(n *yaml.Node) error {
return iterator(n, func(k, v *yaml.Node) (err error) {
var (
wrap = &ComposeNamespace{}
)
if v == nil {
return nodeErr(n, "malformed namespace definition")
}
if err = v.Decode(&wrap); err != nil {
return
}
if k != nil {
if wrap.res.Slug != "" {
return nodeErr(k, "cannot define slug in mapped namespace definition")
}
if !handle.IsValid(k.Value) {
return nodeErr(n, "namespace reference must be a valid handle")
}
// set namespace slug from map key value
wrap.res.Slug = k.Value
}
*wset = append(*wset, wrap)
return
})
}
func (wrap *ComposeNamespace) UnmarshalYAML(n *yaml.Node) error {
if isKind(n, yaml.ScalarNode) {
wrap.ref = n.Value
return nil
}
if !isKind(n, yaml.MappingNode) {
return nodeErr(n, "namespace definition must be a map or scalar")
}
if wrap.res == nil {
wrap.res = &types.Namespace{
// namespaces are enabled by default
Enabled: true,
}
}
return iterator(n, func(k, v *yaml.Node) (err error) {
switch k.Value {
case "slug":
return decodeScalar(v, &wrap.res.Slug)
case "name":
return decodeScalar(v, &wrap.res.Name)
case "enabled":
return decodeScalar(v, &wrap.res.Enabled)
}
return nil
})
}

80
pkg/envoy/yaml/compose_namespace_test.go Normal file
View File

@@ -0,0 +1,80 @@
package yaml
import (
"fmt"
"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"os"
"testing"
)
func TestComposeNamespace_UnmarshalYAML(t *testing.T) {
var (
req = require.New(t)
string = func(src string) (*ComposeNamespace, error) {
w := &ComposeNamespace{}
return w, yaml.Unmarshal([]byte(src), w)
}
file = func(i int) (*Document, error) {
doc := &Document{}
f, err := os.Open(fmt.Sprintf("testdata/compose_namespace_%d.yaml", i))
if err != nil {
return nil, err
}
return doc, yaml.NewDecoder(f).Decode(doc)
}
)
t.Run("empty", func(t *testing.T) {
w, err := string(``)
req.NoError(err)
req.NotNil(w)
req.Nil(w.res)
})
t.Run("simple name", func(t *testing.T) {
w, err := string(`{ name: Test }`)
req.NoError(err)
req.NotNil(w)
req.NotNil(w.res)
req.NotEmpty(w.res.Name)
req.True(w.res.Enabled)
})
t.Run("disabled", func(t *testing.T) {
w, err := string(`{ enabled: false }`)
req.NoError(err)
req.NotNil(w)
req.NotNil(w.res)
req.False(w.res.Enabled)
})
t.Run("compose namespace file 1", func(t *testing.T) {
doc, err := file(1)
req.NoError(err)
req.NotNil(doc)
req.NotNil(doc.compose)
req.Len(doc.compose.namespaces, 1)
req.NotNil(doc.compose.namespaces[0])
req.Equal("CRM", doc.compose.namespaces[0].res.Name)
req.Equal("crm", doc.compose.namespaces[0].res.Slug)
})
t.Run("compose namespace file 2", func(t *testing.T) {
doc, err := file(2)
req.NoError(err)
req.NotNil(doc)
spew.Dump(doc.compose)
req.NotNil(doc.compose)
req.Len(doc.compose.namespaces, 1)
req.NotNil(doc.compose.namespaces[0])
req.Equal("CRM", doc.compose.namespaces[0].res.Name)
req.Equal("crm", doc.compose.namespaces[0].res.Slug)
})
}

164
pkg/envoy/yaml/compose_record.go Normal file
View File

@@ -0,0 +1,164 @@
package yaml
import (
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/handle"
"gopkg.in/yaml.v3"
)
type (
ComposeRecord struct {
res *types.Record `yaml:",inline"`
values ComposeRecordValues `yaml:"values"`
moduleRef string
createdByRef string
updatedByRef string
deletedByRef string
ownedByRef string
}
ComposeRecordSet []*ComposeRecord
ComposeRecordValues struct {
rvs types.RecordValueSet
}
)
// UnmarshalYAML resolves set of record definitions, either sequence or map
//
// When resolving map, key is used as module handle
//
// { module-handle: [ { ... values ... } ] }
// [ { module: module-handle, ... values ... } ]
func (wset *ComposeRecordSet) UnmarshalYAML(n *yaml.Node) error {
return iterator(n, func(k, v *yaml.Node) (err error) {
var (
moduleRef string
)
if k != nil {
// processing mapping node, expecting module handle
if !handle.IsValid(k.Value) {
return nodeErr(k, "module reference must be a valid handle")
}
moduleRef = k.Value
}
if v == nil {
return nodeErr(n, "malformed record definition")
}
if isKind(v, yaml.SequenceNode) {
// multiple records defined
return iterator(v, func(_, r *yaml.Node) error {
var wrap = &ComposeRecord{moduleRef: moduleRef}
if err = r.Decode(&wrap); err != nil {
return err
}
*wset = append(*wset, wrap)
return nil
})
}
if isKind(v, yaml.MappingNode) {
// one record defined
var wrap = &ComposeRecord{moduleRef: moduleRef}
if err = v.Decode(&wrap); err != nil {
return
}
*wset = append(*wset, wrap)
}
return nil
})
}
func (wrap *ComposeRecord) UnmarshalYAML(n *yaml.Node) error {
if !isKind(n, yaml.MappingNode) {
return nodeErr(n, "expecting mapping node for record definition")
}
if wrap.res == nil {
wrap.res = &types.Record{}
}
return iterator(n, func(k, v *yaml.Node) error {
switch k.Value {
case "module":
return decodeRef(v, "module", &wrap.moduleRef)
case "values":
// Use aux structure to decode record values into RVS
aux := ComposeRecordValues{}
if err := v.Decode(&aux); err != nil {
return err
}
wrap.res.Values = aux.rvs
return nil
case "createdAt":
return v.Decode(&wrap.res.CreatedAt)
case "updatedAt":
return v.Decode(&wrap.res.UpdatedAt)
case "deletedAt":
return v.Decode(&wrap.res.DeletedAt)
case "createdBy":
return decodeRef(v, "user", &wrap.createdByRef)
case "updatedBy":
return decodeRef(v, "user", &wrap.updatedByRef)
case "deletedBy":
return decodeRef(v, "user", &wrap.deletedByRef)
case "ownedBy":
return decodeRef(v, "user", &wrap.ownedByRef)
default:
return nodeErr(k, "unsupported key %s used for record definition", k.Value)
}
return nil
})
}
// UnmarshalYAML resolves record values definitioons
//
// { <field name>: ... <scalar value>, .... }
// { <field name>: [ <scalar value> ], .... }
func (wset *ComposeRecordValues) UnmarshalYAML(n *yaml.Node) error {
if !isKind(n, yaml.MappingNode) {
return nodeErr(n, "expecting mapping node for record value definition")
}
wset.rvs = types.RecordValueSet{}
return iterator(n, func(k, v *yaml.Node) error {
if isKind(v, yaml.ScalarNode) {
wset.rvs = append(wset.rvs, &types.RecordValue{
Name: k.Value,
Value: v.Value,
})
return nil
}
if isKind(v, yaml.SequenceNode) {
for i := range v.Content {
if isKind(v, yaml.ScalarNode) {
return nodeErr(n, "expecting scalar node for record value")
}
wset.rvs = append(wset.rvs, &types.RecordValue{
Name: k.Value,
Value: v.Content[i].Value,
Place: uint(i),
})
}
return nil
}
return nodeErr(n, "expecting scalar or sequence node for record value")
})
}

33
pkg/envoy/yaml/compose_record_test.go Normal file
View File

@@ -0,0 +1,33 @@
package yaml
import (
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"testing"
)
func TestComposeRecord_UnmarshalYAML(t *testing.T) {
var (
req = require.New(t)
w *ComposeRecord
err error
parse = func(w *ComposeRecord, src string) (*ComposeRecord, error) {
return w, yaml.Unmarshal([]byte(src), w)
}
)
w, err = parse(&ComposeRecord{}, ``)
req.NoError(err)
req.NotNil(w)
req.Nil(w.res)
w, err = parse(&ComposeRecord{}, `{ values: { foo: bar }, createdBy: foo, updatedAt: 2020-10-10T10:10:00Z, deletedBy: user }`)
req.NoError(err)
req.NotNil(w)
req.NotNil(w.res)
req.NotEmpty(w.res.Values)
req.NotEmpty(w.res.UpdatedAt)
req.Equal("bar", w.res.Values.Get("foo", 0).Value)
}

46
pkg/envoy/yaml/compose_test.go Normal file
View File

@@ -0,0 +1,46 @@
package yaml
import (
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"testing"
)
func TestCompose_UnmarshalYAML(t *testing.T) {
var (
req = require.New(t)
c *compose
err error
parse = func(c *compose, src string) (*compose, error) {
return c, yaml.Unmarshal([]byte(src), c)
}
)
c, err = parse(&compose{}, ``)
req.NoError(err)
req.NotNil(c)
req.Empty(c.namespaces)
c, err = parse(&compose{}, `namespaces: [ { } ]`)
req.NoError(err)
req.NotNil(c)
req.Len(c.namespaces, 1)
c, err = parse(&compose{}, `namespaces: { test: {} }`)
req.NoError(err)
req.NotNil(c)
req.Len(c.namespaces, 1)
c, err = parse(&compose{}, `namespace: { }`)
req.NoError(err)
req.NotNil(c)
req.Len(c.namespaces, 1)
c, err = parse(&compose{}, `namespace: foo`)
req.NoError(err)
req.NotNil(c)
req.Len(c.namespaces, 1)
req.Equal("foo", c.namespaces[0].ref)
}

163
pkg/envoy/yaml/decoder.go Normal file
View File

@@ -0,0 +1,163 @@
package yaml
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"gopkg.in/yaml.v3"
"io"
"os"
"path/filepath"
)
type (
// decoder is a wrapper struct for yaml related methods
decoder struct {
loader loader
}
loader interface {
LoadComposeNamespace()
}
nodeDecoder interface {
DecodeNodes(ctx context.Context, l loader) ([]envoy.Node, error)
}
)
func Decoder(l loader) *decoder {
return &decoder{l}
}
// CanDecodeFile
func (y *decoder) CanDecodeFile(i os.FileInfo) bool {
switch filepath.Ext(i.Name()) {
case "yaml", "yml":
return true
}
return false
}
func (y *decoder) Decode(ctx context.Context, r io.Reader, i os.FileInfo) ([]envoy.Node, error) {
var (
doc = &Document{}
)
if err := yaml.NewDecoder(r).Decode(doc); err != nil {
return nil, fmt.Errorf("failed to decode %s: %w", i.Name(), err)
}
return doc.Decode(ctx, y.loader)
}
//// convert converts the decoded document into a set of envoy nodes
//func (y *decoder) convert(c *Document) ([]envoy.Node, error) {
// nn := make([]envoy.Node, 0, 100)
//
// // In case of namespaces...
// if c.Namespaces != nil {
// nodes, err := y.convertNamespaces(c.Namespaces)
// if err != nil {
// return nil, err
// }
// nn = append(nn, nodes...)
// }
//
// ns := &types.Namespace{}
// if c.Namespace != "" {
// // In case of a namespace to provide dependencies
// ns.Slug = c.Namespace
// ns.Name = c.Namespace
// } else if len(nn) > 0 {
// // Try to fall back to a namespace node
// ns = ((nn[0]).(*envoy.ComposeNamespaceNode)).Ns
// } else {
// // No good; we can't link with a namespace.
// // @note This should be checked when converting Compose resources only.
// // Some resources don't belong to a namespace.
// spew.Dump(c)
// return nil, errors.New("yaml decoder: cannot resolve namespace")
// }
//
// // In case of modules...
// if c.Modules != nil {
// nodes, err := y.convertModules(c.Modules, ns)
// if err != nil {
// return nil, err
// }
// nn = append(nn, nodes...)
// }
//
// if c.Records != nil {
// for modRef, rr := range c.Records {
// // We can define a basic module representation as it will be updated later
// // during validation/runtime
// mod := &types.Module{}
// mod.Handle = modRef
// mod.Name = modRef
//
// nodes, err := y.convertRecords(rr, mod)
// if err != nil {
// return nil, err
// }
// nn = append(nn, nodes...)
// }
// }
//
// return nn, nil
//}
func (y *decoder) convertNamespaces(nss ComposeNamespaceSet) ([]envoy.Node, error) {
nn := make([]envoy.Node, 0, 2)
for _, ns := range nss {
nn = append(nn, &envoy.ComposeNamespaceNode{Ns: ns.res})
// Nested modules
if ns.Modules != nil {
mm, err := y.convertModules(ns.Modules, ns.res)
if err != nil {
return nil, err
}
nn = append(nn, mm...)
}
// @todo nested RBAC
}
return nn, nil
}
func (y *decoder) convertModules(mm ComposeModuleSet, ns *types.Namespace) ([]envoy.Node, error) {
nn := make([]envoy.Node, 0)
for _, m := range mm {
nn = append(nn, &envoy.ComposeModuleNode{
Mod: m.res,
Ns: ns,
})
// @todo nested resources; should there be any?
}
return nn, nil
}
func (y *decoder) convertRecords(rr ComposeRecordSet, m *types.Module) ([]envoy.Node, error) {
// Iterator function for providing records to be imported.
// This doesn't do any validation; that should be handled by other layers.
f := func(f func(record *types.Record) error) error {
//for _, r := range rr {
//err := f(r.res)
//if err != nil {
// return err
//}
//}
return nil
}
return []envoy.Node{&envoy.ComposeRecordNode{Mod: m, Walk: f}}, nil
}

83
pkg/envoy/yaml/doc.go Normal file
View File

@@ -0,0 +1,83 @@
package yaml
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"gopkg.in/yaml.v3"
)
type (
// Document defines the supported yaml structure
Document struct {
compose *compose
}
)
func (doc *Document) UnmarshalYAML(n *yaml.Node) error {
if !isKind(n, yaml.MappingNode) {
// root node kind be mapping
return nodeErr(n, "expecting mapping node")
}
if err := n.Decode(&doc.compose); err != nil {
return err
}
return nil
}
//
func (doc *Document) Decode(ctx context.Context, l loader) ([]envoy.Node, error) {
nn := make([]envoy.Node, 0, 100)
//// In case of namespaces...
//if doc.Namespaces != nil {
// nodes, err := doc.Namespaces.Decode(ctx, l)
// if err != nil {
// return nil, err
// }
// nn = append(nn, nodes...)
//}
//
//ns := &types.Namespace{}
//if doc.Namespace != "" {
// // In case of a namespace to provide dependencies
// ns.Slug = doc.Namespace
// ns.Name = doc.Namespace
//} else if len(nn) > 0 {
// // Try to fall back to a namespace node
// ns = ((nn[0]).(*envoy.ComposeNamespaceNode)).Ns
//} else {
// // No good; we can't link with a namespace.
// // @note This should be checked when converting Compose resources only.
// // Some resources don't belong to a namespace.
// return nil, fmt.Errorf("cannot resolve namespace")
//}
//
//// In case of modules...
//if doc.Modules != nil {
// nodes, err := y.convertModules(doc.Modules, ns)
// if err != nil {
// return nil, err
// }
// nn = append(nn, nodes...)
//}
//
//if doc.Records != nil {
// for modRef, rr := range doc.Records {
// // We can define a basic module representation as it will be updated later
// // during validation/runtime
// mod := &types.Module{}
// mod.Handle = modRef
// mod.Name = modRef
//
// nodes, err := y.convertRecords(rr, mod)
// if err != nil {
// return nil, err
// }
// nn = append(nn, nodes...)
// }
//}
return nn, nil
}

2
pkg/envoy/types/rbac.go → pkg/envoy/yaml/rbac.go
View File

@@ -1,4 +1,4 @@
package types
package yaml
type (
RbacRules map[string][]string

10
pkg/envoy/yaml/testdata/compose_namespace_1.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
namespaces:
crm:
name: CRM
allow:
everyone:
- module.create
- read
deny:
everyone:
- delete

10
pkg/envoy/yaml/testdata/compose_namespace_2.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
namespace:
name: CRM
slug: crm
allow:
everyone:
- module.create
- read
deny:
everyone:
- delete

107
pkg/envoy/yaml/yaml.go Normal file
View File

@@ -0,0 +1,107 @@
package yaml
import (
"fmt"
"github.com/cortezaproject/corteza-server/pkg/handle"
"strings"
"gopkg.in/yaml.v3"
)
func nodeErr(n *yaml.Node, format string, aa ...interface{}) error {
format += " (%d:%d)"
aa = append(aa, n.Line, n.Column)
return fmt.Errorf(format, aa...)
}
// iterator helps iterate over mapping and sequence nodes fairly trivially
func iterator(n *yaml.Node, fn func(*yaml.Node, *yaml.Node) error) error {
if isKind(n, yaml.MappingNode) {
for i := 0; i < len(n.Content); i += 2 {
if err := fn(n.Content[i], n.Content[i+1]); err != nil {
return err
}
}
return nil
}
if isKind(n, yaml.SequenceNode) {
var placeholder *yaml.Node
for i := 0; i < len(n.Content); i++ {
if err := fn(placeholder, n.Content[i]); err != nil {
return err
}
}
return nil
}
return nodeErr(n, "expecting mapping or sequence node")
}
func isKind(n *yaml.Node, tt ...yaml.Kind) bool {
if n != nil {
for _, t := range tt {
if t == n.Kind {
return true
}
}
}
return false
}
// returns true if one or more keys passed in as argument are present under mapping node
func anyKeysPresent(n *yaml.Node, kk ...string) bool {
// make a map with lowercase strings
h := make(map[string]bool)
for i := 0; i < len(kk); i++ {
h[strings.ToLower(kk[i])] = true
}
// compare it with lowercase values
for i := 0; i < len(n.Content); i += 2 {
if !h[n.Content[i+1].Value] {
return false
}
}
return true
}
// findKeyNode returns key node from mapping
// key value checked in lower case
func findKeyNode(n *yaml.Node, key string) *yaml.Node {
// compare it with lowercase values
for i := 0; i < len(n.Content); i += 2 {
if key == n.Content[i].Value {
return n.Content[i+1]
}
}
return nil
}
// Checks validity of ref node and sets the value to given arg ptr
func decodeScalar(n *yaml.Node, val interface{}) error {
if !isKind(n, yaml.ScalarNode) {
return nodeErr(n, "expecting scalar value")
}
return n.Decode(val)
}
// Checks validity of ref node and sets the value to given arg ptr
func decodeRef(n *yaml.Node, refType string, ref *string) error {
if !isKind(n, yaml.ScalarNode) {
return nodeErr(n, "%s reference must be scalar", refType)
}
if !handle.IsValid(n.Value) {
return nodeErr(n, "%s reference must be a valid handle", refType)
}
*ref = n.Value
return nil
}