From b1d4531748e4c55d22ed2118e914704279d1901b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Thu, 16 Feb 2023 15:46:12 +0100 Subject: [PATCH] Add base implementation of reworked dep. management --- server/pkg/envoyx/dep.go | 471 +++++++++++++++++++++++++++++++++ server/pkg/envoyx/dep_test.go | 389 +++++++++++++++++++++++++++ server/pkg/envoyx/envoy.go | 212 +++++++++++++++ server/pkg/envoyx/io.go | 109 ++++++++ server/pkg/envoyx/node.go | 283 ++++++++++++++++++++ server/pkg/envoyx/node_test.go | 118 +++++++++ server/pkg/envoyx/store.go | 44 +++ 7 files changed, 1626 insertions(+) create mode 100644 server/pkg/envoyx/dep.go create mode 100644 server/pkg/envoyx/dep_test.go create mode 100644 server/pkg/envoyx/envoy.go create mode 100644 server/pkg/envoyx/io.go create mode 100644 server/pkg/envoyx/node.go create mode 100644 server/pkg/envoyx/node_test.go create mode 100644 server/pkg/envoyx/store.go diff --git a/server/pkg/envoyx/dep.go b/server/pkg/envoyx/dep.go new file mode 100644 index 000000000..b9740156a --- /dev/null +++ b/server/pkg/envoyx/dep.go @@ -0,0 +1,471 @@ +package envoyx + +type ( + // depGraph provides a collection of optionally connected subgraphs + // + // Each subgraph is dedicated for a specific node scope. + // A subgraph with scoped nodes may be connected to the subgraph with no + // defined scope. + depGraph struct { + graphs []*depSubgraph + } + + // depSubgraph provides a bi-directional dependency graph + depSubgraph struct { + scope Scope + + nodes []*depNode + nodeMap map[*Node]*depNode + + parentSubgraphs map[*depSubgraph]bool + childSubgraphs map[*depSubgraph]bool + } + + // depNode provides a wrapper around the resource for eas of use within the dep. graph + depNode struct { + Node *Node + + parents map[*depNode]bool + children map[*depNode]bool + + // Index to keep track of missing references at build time + missingReferences map[string]Ref + } +) + +// BuildDepGraph constructs a dependency graph from the provided nodes +// +// We firstly group the nodes by scope, then build a subgraph for each scope, +// and lastly merge the subgraphs into a single graph. +func BuildDepGraph(nn ...*Node) (out *depGraph) { + scopes := scopeNodes(nn...) + + aux := make([]*depSubgraph, 0, len(scopes)) + for _, ss := range scopes { + aux = append(aux, buildDepSubgraph(ss)) + } + + return buildDepGraph(aux...) +} + +// scopeNodes groups the nodes by the scope +// +// The function returns a slice of NodeSet where each NodeSet only contains +// nodes of the same scope (or no scope if none defined). +func scopeNodes(nn ...*Node) (out []NodeSet) { + type ( + // Defining a little wrapper around the NodeSet so we can use pointers + scopeWrap struct { + nn NodeSet + } + ) + + var ( + // resource type -> identifier -> wrap + scopes = make(map[string]map[string]*scopeWrap) + + // Holds the nodes with no defined scope + empty NodeSet + ) + + // Bucket nodes into scopes + // + // Use maps to make the process as efficient as possible. + for _, n := range nn { + // Empty scopes are a special case + if n.Scope.IsEmpty() { + empty = append(empty, n) + continue + } + + // New resource type + if _, ok := scopes[n.Scope.ResourceType]; !ok { + scopes[n.Scope.ResourceType] = make(map[string]*scopeWrap, 4) + } + + // Check if the identifiers of the current node scope exist in the index. + // If they do, update the wrap struct by adding this node, then register + // the same wrap struct to all other identifiers -- this allows for some + // recovery in case some resource uses a subset of one but not the other. + hasIdent := false + firstIdent := "" + for _, i := range n.Scope.Identifiers.Slice { + hasIdent = hasIdent || scopes[n.Scope.ResourceType][i] != nil + if hasIdent { + firstIdent = i + break + } + } + + if !hasIdent { + // Not registered yet, create a new wrap for all of the identifiers + w := &scopeWrap{nn: append(make(NodeSet, 0, 10), n)} + for _, i := range n.Scope.Identifiers.Slice { + scopes[n.Scope.ResourceType][i] = w + } + } else { + // Already registered; update it and add missing identifiers + w := scopes[n.Scope.ResourceType][firstIdent] + w.nn = append(w.nn, n) + for _, i := range n.Scope.Identifiers.Slice { + scopes[n.Scope.ResourceType][i] = w + } + } + } + + // Unpack the map into a slice of NodeSet the return expects + // + // @note since all identifiers are registered, the wraps would appear + // duplicated so we need to filter them a bit + out = make([]NodeSet, 0, 10) + if len(empty) > 0 { + out = append(out, empty) + } + seen := make(map[*scopeWrap]bool) + for _, s := range scopes { + for _, ss := range s { + if seen[ss] { + continue + } + seen[ss] = true + out = append(out, ss.nn) + } + } + + return out +} + +// buildDepSubgraph constructs a dep. subgraph from the provided nodes +// +// The function returns a bidirectional graph of the nodes where the parent +// represents the dependency of the current resource (a namespace would be +// a parent of a module). +// +// The build process indexes some data for optimal operations down the line. +func buildDepSubgraph(nn NodeSet) (out *depSubgraph) { + out = &depSubgraph{ + scope: nn[0].Scope, + + nodes: make([]*depNode, len(nn)), + nodeMap: make(map[*Node]*depNode, len(nn)), + + parentSubgraphs: make(map[*depSubgraph]bool), + childSubgraphs: make(map[*depSubgraph]bool), + } + + byIdentifier := make(map[string]map[string]*depNode, 8) + byNode := make(map[*Node]*depNode, len(nn)) + + // 1. index all of the nodes in a map so we can trivially connect them later + for i, _n := range nn { + n := _n + + // Function blindly trusts it will be called with the correct data. + // @todo consider implementing this check but make sure all of the decoders + // correctly set the scopes. + // if !n.Scope.Equals(nn[0].Scope) { + // panic("invalid state: subgraphs can only be constructed with nodes from the same scope") + // } + + aux := &depNode{ + Node: n, + + // Keep track of missing references so we can figure them out optimally + missingReferences: make(map[string]Ref), + + parents: make(map[*depNode]bool), + children: make(map[*depNode]bool), + } + + for field, ref := range n.References { + aux.missingReferences[field] = ref + } + + byNode[n] = aux + out.nodes[i] = aux + out.nodeMap[n] = aux + + if _, ok := byIdentifier[n.ResourceType]; !ok { + byIdentifier[n.ResourceType] = make(map[string]*depNode, 8) + } + + for _, i := range n.Identifiers.Slice { + byIdentifier[n.ResourceType][i] = byNode[n] + } + } + + // 2. link up the node with it's dependencies + for _, _n := range out.nodes { + n := _n + + for field, ref := range n.Node.References { + resource := ref.ResourceType + found := false + + for _, i := range ref.Identifiers.Slice { + ch, ok := byIdentifier[resource][i] + found = found || ok + + if ok { + delete(n.missingReferences, field) + n.parents[ch] = true + ch.children[n] = true + break + } + } + } + } + + return +} + +// buildDepGraph constructs a dependency graph from the given set of subgraphs +// +// The subgraphs can be connected in case a scoped node would reference a +// unscoped node (unscoped nodes can not reference scoped nodes, nor can nodes +// from different scopes -- unneeded and removes some complexity). +func buildDepGraph(gg ...*depSubgraph) (out *depGraph) { + out = &depGraph{ + graphs: make([]*depSubgraph, 0, len(gg)), + } + + // Get the unscoped graph + // + // For now, we can only xref to unscoped graphs. + // This might need to be generalized. + unscopedG := unscopedSubgraph(gg...) + + // Iterate all graphs and try to resolve unresolved deps + for _, g := range gg { + out.graphs = append(out.graphs, g) + + if unscopedG == nil { + continue + } + + // Get nodes with missing references + missingNodes := g.nodesWithMissingRefs() + if len(missingNodes) == 0 { + continue + } + + // Try to resolve missing refs using the unscoped graph + for _, _mn := range missingNodes { + mn := _mn + + for field, ref := range mn.missingReferences { + n := depNodeForRef(ref, unscopedG.nodes...) + if n == nil { + continue + } + + mn.parents[n] = true + n.children[mn] = true + + delete(mn.missingReferences, field) + if len(mn.missingReferences) == 0 { + mn.missingReferences = nil + } + } + } + + // xref the subgraphs + g.parentSubgraphs[unscopedG] = true + unscopedG.childSubgraphs[g] = true + } + + return +} + +// Graph traversal functions + +// Roots returns all nodes which are considered as root resources based on the current state +// +// For the most part, these are all resources with no parent resources. +// If all resources define parents, then some home brew logic is ran +func (g depGraph) Roots() (out NodeSet) { + for _, sg := range g.graphs { + out = append(out, sg.Roots()...) + } + return +} + +// ParentForRef returns a parent node of n which matches ref (nil if none) +func (g depGraph) ParentForRef(n *Node, ref Ref) (out *Node) { + for _, sg := range g.graphs { + out = sg.ParentForRef(n, ref) + if out != nil { + return + } + } + return +} + +// ChildrenForResourceType returns child nodes of n which match the resource type +func (g depGraph) ChildrenForResourceType(n *Node, rt string) (out NodeSet) { + for _, sg := range g.graphs { + out = sg.ChildrenForResourceType(n, rt) + if out != nil { + return + } + } + return +} + +// Children returns all child nodes of n +func (g depGraph) Children(n *Node) (out NodeSet) { + for _, sg := range g.graphs { + out = sg.Children(n) + if out != nil { + return + } + } + return +} + +// MissingRegs returns a slice of all refs that are requested but not found in the graph +func (g depGraph) MissingRefs() (out []map[string]Ref) { + for _, sg := range g.graphs { + for _, n := range sg.nodes { + if len(n.missingReferences) == 0 { + continue + } + out = append(out, n.missingReferences) + } + } + + return +} + +func (g depGraph) allNodes() (out []*depNode) { + for _, sg := range g.graphs { + out = append(out, sg.nodes...) + } + + return +} + +// Roots returns all nodes which are considered as root resources based on the current state +// +// For the most part, these are all resources with no parent resources. +// If all resources define parents, then some home brew logic is ran +// +// @todo when we add more resources, we might need to expand this; for now +// it should work just fine. +func (g depSubgraph) Roots() (out NodeSet) { + for _, n := range g.nodes { + if needyResources[n.Node.ResourceType] { + continue + } + out = append(out, n.Node) + } + + if len(out) != 0 { + return + } + + for _, n := range g.nodes { + if superNeedyResources[n.Node.ResourceType] { + continue + } + out = append(out, n.Node) + } + + return +} + +// ParentForRef returns a parent node of n which matches ref (nil if none) +func (g depSubgraph) ParentForRef(n *Node, ref Ref) *Node { + return NodeForRef(ref, g.parent(n)...) +} + +// ChildrenForResourceType returns child nodes of n which match the resource type +func (g depSubgraph) ChildrenForResourceType(n *Node, rt string) (out NodeSet) { + out = g.Children(n) + out = NodesForResourceType(rt, out...) + return +} + +// Children returns all child nodes of n +func (g depSubgraph) Children(n *Node) (out NodeSet) { + aux, ok := g.nodeMap[n] + if !ok { + return + } + + for n := range aux.children { + out = append(out, n.Node) + } + return +} + +func (g depSubgraph) parent(n *Node) (out NodeSet) { + aux, ok := g.nodeMap[n] + if !ok { + return + } + + for n := range aux.parents { + out = append(out, n.Node) + } + return +} + +// Utility functions + +func (g *depSubgraph) nodesWithMissingRefs() (out []*depNode) { + out = make([]*depNode, 0, 3) + for _, n := range g.nodes { + if len(n.missingReferences) > 0 { + out = append(out, n) + } + } + + return +} + +func unscopedSubgraph(gg ...*depSubgraph) *depSubgraph { + for _, g := range gg { + if g.scope.IsEmpty() { + return g + } + } + + return nil +} + +// depNodeForRef returns a node which matches the ref (nil if none) +func depNodeForRef(ref Ref, nn ...*depNode) (out *depNode) { + for _, n := range nn { + if n.Node.ResourceType != ref.ResourceType { + continue + } + + if n.Node.Identifiers.HasIntersection(ref.Identifiers) { + return n + } + } + + return +} + +// depNodesByResourceType returns a map where key is resource type, value slice of nodes +func depNodesByResourceType(nn ...*depNode) (out map[string][]*depNode) { + out = make(map[string][]*depNode, 4) + for _, n := range nn { + out[n.Node.ResourceType] = append(out[n.Node.ResourceType], n) + } + + return +} + +// unpackDepNodes extracts envoy Nodes from the dep. nodes +// +// The function does no validation nor filtering for nil values. +func unpackDepNodes(nn ...*depNode) (out NodeSet) { + out = make(NodeSet, 0, len(nn)) + for _, n := range nn { + out = append(out, n.Node) + } + return out +} diff --git a/server/pkg/envoyx/dep_test.go b/server/pkg/envoyx/dep_test.go new file mode 100644 index 000000000..9e39d7fc4 --- /dev/null +++ b/server/pkg/envoyx/dep_test.go @@ -0,0 +1,389 @@ +package envoyx + +import ( + "fmt" + "testing" + + "github.com/cortezaproject/corteza/server/compose/types" + systemTypes "github.com/cortezaproject/corteza/server/system/types" + "github.com/stretchr/testify/require" +) + +func TestScopeNodes(t *testing.T) { + req := require.New(t) + + t.Run("none scoped", func(t *testing.T) { + a := &Node{} + b := &Node{} + c := &Node{} + + ss := scopeNodes(a, b, c) + req.Len(ss, 1) + req.Len(ss[0], 3) + req.Contains(ss[0], a) + }) + + t.Run("all same scope", func(t *testing.T) { + a := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("a"), + }, + } + b := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("a"), + }, + } + c := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("a"), + }, + } + + ss := scopeNodes(a, b, c) + req.Len(ss, 1) + req.Len(ss[0], 3) + req.Contains(ss[0], a) + }) + + t.Run("mixed scope", func(t *testing.T) { + none1 := &Node{} + none2 := &Node{} + + a1 := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("a"), + }, + } + a2 := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("a"), + }, + } + + b1 := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("b"), + }, + } + b2 := &Node{ + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("b"), + }, + } + + ss := scopeNodes(none1, none2, a1, a2, b1, b2) + req.Len(ss, 3) + + req.Len(ss[0], 2) + req.Len(ss[1], 2) + req.Len(ss[2], 2) + + req.True(ss[0][0].Scope.Identifiers.HasIntersection(ss[0][1].Scope.Identifiers)) + req.True(ss[1][0].Scope.Identifiers.HasIntersection(ss[1][1].Scope.Identifiers)) + req.True(ss[2][0].Scope.Identifiers.HasIntersection(ss[2][1].Scope.Identifiers)) + }) +} + +func TestDepRoots(t *testing.T) { + req := require.New(t) + + t.Run("no nodes", func(t *testing.T) { + gg := BuildDepGraph() + req.Len(gg.Roots(), 0) + }) + + t.Run("single non-needy node", func(t *testing.T) { + a1 := &Node{ + ResourceType: systemTypes.UserResourceType, + Identifiers: MakeIdentifiers("A.1"), + } + + gg := BuildDepGraph(a1) + + req.Len(gg.Roots(), 1) + req.Contains(gg.Roots(), a1) + }) + + t.Run("simple compose ns-mod dep setup", func(t *testing.T) { + a1 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("A.1"), + } + b1 := &Node{ + ResourceType: types.ModuleResourceType, + Identifiers: MakeIdentifiers("B.1"), + References: map[string]Ref{ + "NamespaceID": a1.ToRef(), + }, + } + + gg := BuildDepGraph(a1, b1) + + req.Len(gg.Roots(), 1) + req.Contains(gg.Roots(), a1) + }) + + t.Run("simple compose mod-mod field dep setup", func(t *testing.T) { + a1 := &Node{ + ResourceType: types.ModuleResourceType, + Identifiers: MakeIdentifiers("A.1"), + } + b1 := &Node{ + ResourceType: types.ModuleFieldResourceType, + Identifiers: MakeIdentifiers("B.1"), + References: map[string]Ref{ + "NamespaceID": a1.ToRef(), + }, + } + + gg := BuildDepGraph(a1, b1) + + req.Len(gg.Roots(), 1) + req.Contains(gg.Roots(), a1) + }) +} + +func TestDepTraversal(t *testing.T) { + req := require.New(t) + + ns1 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + }, + } + ns2 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns2"), + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns2"), + }, + } + + ns1mod1 := &Node{ + ResourceType: types.ModuleResourceType, + Identifiers: MakeIdentifiers("mod"), + References: map[string]Ref{ + "NamespaceID": ns1.ToRef(), + }, + Scope: ns1.Scope, + } + ns1mod1f1 := &Node{ + ResourceType: types.ModuleFieldResourceType, + Identifiers: MakeIdentifiers("f1"), + References: map[string]Ref{ + "NamespaceID": ns1.ToRef(), + "ModuleID": ns1mod1.ToRef(), + }, + Scope: ns1.Scope, + } + + ns2mod1 := &Node{ + ResourceType: types.ModuleResourceType, + Identifiers: MakeIdentifiers("mod"), + References: map[string]Ref{ + "NamespaceID": ns2.ToRef(), + }, + Scope: ns2.Scope, + } + ns2mod1f1 := &Node{ + ResourceType: types.ModuleFieldResourceType, + Identifiers: MakeIdentifiers("f1"), + References: map[string]Ref{ + "NamespaceID": ns2.ToRef(), + "ModuleID": ns2mod1.ToRef(), + }, + Scope: ns2.Scope, + } + + gg := BuildDepGraph(ns1, ns2, ns1mod1, ns1mod1f1, ns2mod1, ns2mod1f1) + + t.Run("children of namespace root", func(t *testing.T) { + cc := gg.Children(ns1) + + req.Len(cc, 2) + req.Contains(cc, ns1mod1) + req.Contains(cc, ns1mod1f1) + }) + + t.Run("children module", func(t *testing.T) { + cc := gg.Children(ns1mod1) + + req.Len(cc, 1) + req.Contains(cc, ns1mod1f1) + }) + + t.Run("parent by ref missing", func(t *testing.T) { + p := gg.ParentForRef(ns1mod1, Ref{ResourceType: types.NamespaceResourceType, Identifiers: MakeIdentifiers("asdf"), Scope: ns1.Scope}) + + req.Nil(p) + }) + + t.Run("parent by ref wrong scope", func(t *testing.T) { + p := gg.ParentForRef(ns1mod1, Ref{ResourceType: types.NamespaceResourceType, Identifiers: ns2.Identifiers, Scope: ns2.Scope}) + + req.Nil(p) + }) + + t.Run("parent by ref", func(t *testing.T) { + p := gg.ParentForRef(ns1mod1, Ref{ResourceType: types.NamespaceResourceType, Identifiers: ns1.Identifiers, Scope: ns1.Scope}) + + req.NotNil(p) + req.Equal(ns1, p) + }) +} + +func TestDepXLinking(t *testing.T) { + req := require.New(t) + + s1 := &Node{ + ResourceType: systemTypes.UserResourceType, + Identifiers: MakeIdentifiers("u1"), + } + s2 := &Node{ + ResourceType: systemTypes.UserResourceType, + Identifiers: MakeIdentifiers("u2"), + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + }, + } + + ns1 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + References: map[string]Ref{ + "S1": s1.ToRef(), + }, + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + }, + } + ns2 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns2"), + References: map[string]Ref{ + "S2": s2.ToRef(), + }, + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns2"), + }, + } + _ = ns2 + + t.Run("scoped node access unscoped ref", func(t *testing.T) { + gg := BuildDepGraph(s1, s2, ns1) + req.NotNil(gg.ParentForRef(ns1, s1.ToRef())) + }) + + t.Run("does not have missing refs", func(t *testing.T) { + gg := BuildDepGraph(s1, s2, ns1) + mm := gg.MissingRefs() + req.Len(mm, 0) + }) + + t.Run("scoped node prevented ref of different scope", func(t *testing.T) { + gg := BuildDepGraph(s1, s2, ns2) + req.Nil(gg.ParentForRef(ns2, s2.ToRef())) + }) + + t.Run("has missing refs", func(t *testing.T) { + gg := BuildDepGraph(s1, s2, ns2) + mm := gg.MissingRefs() + req.Len(mm, 1) + }) +} + +// goos: linux +// goarch: amd64 +// pkg: github.com/cortezaproject/corteza/server/pkg/envoyx +// cpu: Intel(R) Core(TM) i7-8750H CPU @ 2.20GHz +// BenchmarkDepGraphConstruction-12 826 1495619 ns/op 1633445 B/op 6168 allocs/op +// PASS +func BenchmarkDepGraphConstruction(b *testing.B) { + ns1 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns1"), + }, + } + ns2 := &Node{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns2"), + Scope: Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: MakeIdentifiers("ns2"), + }, + } + + ns1mod1 := &Node{ + ResourceType: types.ModuleResourceType, + Identifiers: MakeIdentifiers("mod"), + References: map[string]Ref{ + "NamespaceID": ns1.ToRef(), + }, + Scope: ns1.Scope, + } + ns1mod1f1 := &Node{ + ResourceType: types.ModuleFieldResourceType, + Identifiers: MakeIdentifiers("f1"), + References: map[string]Ref{ + "NamespaceID": ns1.ToRef(), + "ModuleID": ns1mod1.ToRef(), + }, + Scope: ns1.Scope, + } + + ns2mod1 := &Node{ + ResourceType: types.ModuleResourceType, + Identifiers: MakeIdentifiers("mod"), + References: map[string]Ref{ + "NamespaceID": ns2.ToRef(), + }, + Scope: ns2.Scope, + } + ns2mod1f1 := &Node{ + ResourceType: types.ModuleFieldResourceType, + Identifiers: MakeIdentifiers("f1"), + References: map[string]Ref{ + "NamespaceID": ns2.ToRef(), + "ModuleID": ns2mod1.ToRef(), + }, + Scope: ns2.Scope, + } + + qwerty := make(NodeSet, 0, 1000-6) + for i := 0; i < 1000-6; i++ { + qwerty = append(qwerty, &Node{ + ResourceType: types.ModuleFieldResourceType, + Identifiers: MakeIdentifiers(fmt.Sprintf("gg_f_%d", i)), + References: map[string]Ref{ + "NamespaceID": ns1.ToRef(), + "ModuleID": ns1mod1.ToRef(), + }, + Scope: ns1.Scope, + }) + } + + qwerty = append(qwerty, NodeSet{ns1, ns2, ns1mod1, ns1mod1f1, ns2mod1, ns2mod1f1}...) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + BuildDepGraph(qwerty...) + } +} diff --git a/server/pkg/envoyx/envoy.go b/server/pkg/envoyx/envoy.go new file mode 100644 index 000000000..f0e3b1373 --- /dev/null +++ b/server/pkg/envoyx/envoy.go @@ -0,0 +1,212 @@ +package envoyx + +import ( + "context" + "fmt" +) + +type ( + service struct { + decoders map[decodeType][]Decoder + + encoders map[encodeType][]Encoder + preparers map[encodeType][]Preparer + } + + // Traverser provides a structure which can be used to traverse the node's deps + Traverser interface { + // ParentForRef returns the parent of the provided node which matches the ref + // + // If no parent is found, nil is returned. + ParentForRef(*Node, Ref) *Node + + // ChildrenForResourceType returns the children of the provided node which + // match the provided resource type + ChildrenForResourceType(*Node, string) NodeSet + + // Children returns all of the children of the provided node + Children(*Node) NodeSet + } + + Preparer interface { + // Prepare performs generic preprocessing on the provided nodes + // + // The function is called for every resource type where all of the nodes of + // that resource type are passed as the argument. + Prepare(context.Context, EncodeParams, string, NodeSet) error + } + + Encoder interface { + // Encode encodes the data + // + // The function receives a set of root-level nodes (with no parent dependencies) + // and a Traverser it can use to handle all of the child nodes. + Encode(context.Context, EncodeParams, string, NodeSet, Traverser) (err error) + } + + PrepareEncoder interface { + Preparer + Encoder + } + + Decoder interface { + // Decode returns a set of Nodes extracted based on the provided definition + Decode(ctx context.Context, p DecodeParams) (out NodeSet, err error) + } + + DecodeParams struct { + Type decodeType + Params map[string]any + Config DecoderConfig + Filter map[string]ResourceFilter + } + DecoderConfig struct{} + + EncodeParams struct { + Type encodeType + Params map[string]any + Config EncoderConfig + } + EncoderConfig struct { + OnExisting mergeAlg + + PreferredTimeLayout string + PreferredTimezone string + } + + ResourceFilter struct { + Identifiers Identifiers + Refs map[string]Ref + + Limit uint + Scope Scope + } + + decodeType string + encodeType string + mergeAlg int +) + +var ( + global *service +) + +const ( + OnConflictReplace mergeAlg = iota + OnConflictSkip + OnConflictPanic + // OnConflictMergeLeft mergeAlg = "mergeLeft" + // OnConflictMergeRight mergeAlg = "mergeRight" + + DecodeTypeURI decodeType = "uri" + DecodeTypeStore decodeType = "store" + + EncodeTypeURI encodeType = "uri" + EncodeTypeStore encodeType = "store" + EncodeTypeIo encodeType = "io" +) + +// New initializes a new Envoy service +func New() *service { + return &service{} +} + +// SetGlobal sets the global envoy service +func SetGlobal(n *service) { + global = n +} + +// Service gets the global envoy service +func Service() *service { + if global == nil { + panic("global service not defined") + } + + return global +} + +// Decode returns a set of envoy Nodes based on the given decode params +func (svc *service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err error) { + err = p.validate() + if err != nil { + return + } + + switch p.Type { + case DecodeTypeURI: + return svc.decodeUri(ctx, p) + case DecodeTypeStore: + return svc.decodeStore(ctx, p) + } + + return +} + +// Encode encodes Corteza resources bases on the provided encode params +// +// use the BuildDepGraph function to build the default dependency graph. +func (svc *service) Encode(ctx context.Context, p EncodeParams, dg *depGraph) (err error) { + err = p.validate() + if err != nil { + return + } + + switch p.Type { + case EncodeTypeStore: + return svc.encodeStore(ctx, dg, p) + case EncodeTypeIo: + return svc.encodeIo(ctx, dg, p) + + } + return +} + +func (svc *service) AddDecoder(t decodeType, dd ...Decoder) { + if svc.decoders == nil { + svc.decoders = make(map[decodeType][]Decoder) + } + svc.decoders[t] = append(svc.decoders[t], dd...) +} + +func (svc *service) AddEncoder(t encodeType, ee ...Encoder) { + if svc.encoders == nil { + svc.encoders = make(map[encodeType][]Encoder) + } + svc.encoders[t] = append(svc.encoders[t], ee...) +} + +func (svc *service) AddPreparer(t encodeType, pp ...Preparer) { + if svc.preparers == nil { + svc.preparers = make(map[encodeType][]Preparer) + } + svc.preparers[t] = append(svc.preparers[t], pp...) +} + +// Utility functions + +func (p DecodeParams) validate() (err error) { + switch p.Type { + case DecodeTypeURI: + _, ok := p.Params["uri"] + if !ok { + return fmt.Errorf("uhoh, no uri provided") + } + + case DecodeTypeStore: + + } + + // @todo... + + return +} + +func (p EncodeParams) validate() (err error) { + switch p.Type { + // @todo... + } + + // @todo... + + return +} diff --git a/server/pkg/envoyx/io.go b/server/pkg/envoyx/io.go new file mode 100644 index 000000000..2dfff79ad --- /dev/null +++ b/server/pkg/envoyx/io.go @@ -0,0 +1,109 @@ +package envoyx + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func (nvyx *service) decodeUri(ctx context.Context, p DecodeParams) (nn NodeSet, err error) { + aUri, ok := p.Params["uri"] + if !ok { + err = fmt.Errorf("cannot decode URI: no uri parameter provided") + return + } + + uri, ok := aUri.(string) + if !ok { + err = fmt.Errorf("cannot decode URI: uri should be string encoded: got %v", aUri) + return + } + + pp := strings.Split(uri, "://") + + proto := pp[0] + rest := pp[1] + + switch proto { + case "file": + fileInfo, err := os.Stat(rest) + if err != nil { + return nil, err + } + + if fileInfo.IsDir() { + // decode whole directory + return nvyx.decodeDirectory(ctx, p, rest) + } + + // decode specific file + return nvyx.decodeFile(ctx, p, rest) + + default: + err = fmt.Errorf("unsupported URI protocol %s", proto) + return + } +} + +func (nvyx *service) encodeIo(ctx context.Context, dg *depGraph, p EncodeParams) (err error) { + for rt, nn := range NodesByResourceType(dg.Roots()...) { + for _, se := range nvyx.encoders[EncodeTypeIo] { + err = se.Encode(ctx, p, rt, OmitPlaceholderNodes(nn...), dg) + if err != nil { + return + } + } + } + + return +} + +func (nvyx *service) decodeDirectory(ctx context.Context, p DecodeParams, path string) (nn NodeSet, err error) { + return nn, filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // @todo consider supporting nested directories + if info.IsDir() { + return nil + } + + aux, err := nvyx.decodeFile(ctx, p, path) + if err != nil { + return err + } + + nn = append(nn, aux...) + return nil + }) +} + +func (nvyx *service) decodeFile(ctx context.Context, p DecodeParams, path string) (nn NodeSet, err error) { + var aux NodeSet + f, err := os.Open(path) + if err != nil { + return + } + defer f.Close() + + p.Params["stream"] = f + + for _, d := range nvyx.decoders[DecodeTypeURI] { + aux, err = d.Decode(ctx, p) + if err != nil { + return + } + nn = append(nn, aux...) + + _, err = f.Seek(0, 0) + if err != nil { + return + } + } + + return +} diff --git a/server/pkg/envoyx/node.go b/server/pkg/envoyx/node.go new file mode 100644 index 000000000..b167a64c7 --- /dev/null +++ b/server/pkg/envoyx/node.go @@ -0,0 +1,283 @@ +package envoyx + +import ( + "strconv" + + "github.com/spf13/cast" +) + +type ( + // Node is a wrapper around a Corteza resource for use within Envoy + Node struct { + Resource resource + + ResourceType string + Identifiers Identifiers + References map[string]Ref + Scope Scope + + // Placeholders are resources which were added to help resolve missing deps + Placeholder bool + } + + NodeSet []*Node + + resource interface { + SetValue(name string, pos uint, value any) error + GetValue(name string, pos uint) (any, error) + GetID() uint64 + } + + Identifiers struct { + Slice []string + Index map[string]bool + } + + // Scope lets us group nodes based on some common context + // + // Scope is primarily used to scope low-code applications to denote to what + // namespace a specific module reference belongs to. + // In the previous version this was referred to as reference constraints; + // This is the same but different. + // + // When constructing dependency graphs, nodes with the same scope are grouped together. + // Nodes from the same scope can reference each other. + // Nodes from a defined scope can reference nodes from an undefined scope, + // but not the other way around. + Scope struct { + ResourceType string + Identifiers Identifiers + } + + // Ref defines a reference to a different resource + // + // The reference only holds if all three parts match -- the resource type, + // there is an intersection between the identifiers, and the scope matches. + Ref struct { + ResourceType string + Identifiers Identifiers + Scope Scope + } +) + +// MakeIdentifiers initializes an Identifiers instance from the given slice +func MakeIdentifiers(ii ...any) (out Identifiers) { + return Identifiers{}.Add(ii...) +} + +// NodesByResourceType returns Nodes grouped by their resource type +func NodesByResourceType(nn ...*Node) (out map[string]NodeSet) { + out = make(map[string]NodeSet, len(nn)/2) + for _, n := range nn { + out[n.ResourceType] = append(out[n.ResourceType], n) + } + + return +} + +// NodesForResourceType returns which belong to the given resource type +func NodesForResourceType(rt string, nn ...*Node) (out NodeSet) { + return NodesByResourceType(nn...)[rt] +} + +// NodeForRef returns the Node that matches the given ref +func NodeForRef(ref Ref, nn ...*Node) (out *Node) { + for _, n := range nn { + if !n.Scope.Equals(ref.Scope) { + continue + } + + if n.ResourceType != ref.ResourceType { + continue + } + + if n.Identifiers.HasIntersection(ref.Identifiers) { + return n + } + } + + return +} + +func OmitPlaceholderNodes(nn ...*Node) (out NodeSet) { + out = make(NodeSet, 0, len(nn)) + for _, n := range nn { + if n.Placeholder { + continue + } + out = append(out, n) + } + return +} + +func MergeRefs(a, b map[string]Ref) (c map[string]Ref) { + c = make(map[string]Ref) + + for k, v := range a { + c[k] = v + } + for k, v := range b { + c[k] = v + } + + return +} + +// MergeIdents merges the two identifiers and returns a new one +func MergeIdents(a, b Identifiers) (cc Identifiers) { + cc = Identifiers{ + Index: make(map[string]bool, 2), + } + + for a := range a.Index { + cc.Index[a] = true + } + for b := range b.Index { + cc.Index[b] = true + } + + for c := range cc.Index { + cc.Slice = append(cc.Slice, c) + } + + return +} + +func (n Node) ToRef() Ref { + return Ref{ + ResourceType: n.ResourceType, + Identifiers: n.Identifiers, + Scope: n.Scope, + } +} + +func (r Ref) Idents() (ints []uint64, rest []string) { + return r.Identifiers.Idents() +} + +// ResourceFilter returns a filter which would match the referenced resource +func (r Ref) ResourceFilter() (out map[string]ResourceFilter) { + out = make(map[string]ResourceFilter) + out[r.ResourceType] = ResourceFilter{ + Identifiers: r.Identifiers, + Scope: r.Scope, + + // A ref would point to a single resource. + // Don't set the limit so we can error out on ambiguity. + // Limit: 1, + } + + return +} + +func (a Scope) Equals(b Scope) bool { + if a.IsEmpty() && b.IsEmpty() { + return true + } + + if a.ResourceType != b.ResourceType { + return false + } + + return a.Identifiers.HasIntersection(b.Identifiers) +} + +func (s Scope) IsEmpty() bool { + return s.ResourceType == "" && len(s.Identifiers.Slice) == 0 +} + +// Add adds the given values to the identifier +func (ii Identifiers) Add(vv ...any) (out Identifiers) { + if ii.Index == nil { + ii.Index = make(map[string]bool, len(vv)) + ii.Slice = make([]string, 0, len(vv)) + } + + for _, v := range vv { + switch casted := v.(type) { + case string: + if casted == "" { + continue + } + case uint64, uint, int, int64: + if casted == 0 { + continue + } + } + + if c, ok := v.(Identifiers); ok { + ii = ii.Merge(c) + continue + } + + aux := cast.ToString(v) + if aux == "" || aux == "0" { + continue + } + + ii.Slice = append(ii.Slice, aux) + ii.Index[aux] = true + } + + return ii +} + +// Idents returns a slice of numeric and text identifiers +func (ii Identifiers) Idents() (ints []uint64, rest []string) { + var aux uint64 + var err error + + for _, i := range ii.Slice { + aux, err = cast.ToUint64E(i) + if err != nil { + rest = append(rest, i) + } else { + ints = append(ints, aux) + } + } + + return +} + +// Intersection returns a slice of identifiers which are in an intersection +func (aa Identifiers) Intersection(bb Identifiers) (out []string) { + for _, b := range bb.Slice { + if aa.Index[b] { + out = append(out, b) + } + } + + return +} + +// HasIntersection returns true if the two identifiers define an intersection +func (aa Identifiers) HasIntersection(bb Identifiers) bool { + if len(aa.Slice) == 0 && len(bb.Slice) == 0 { + return true + } + + return len(aa.Intersection(bb)) > 0 +} + +// Merge merges the two identifiers and returns a new one +// @todo deprecate this; use MergeIdents instead +func (aa Identifiers) Merge(bb Identifiers) (cc Identifiers) { + return MergeIdents(aa, bb) +} + +// FriendlyIdentifier returns the best available identifier +// +// If any non-ID identifiers are available, it uses the first one. +// If no non-ID identifiers are available, it returns the first ID. +func (ii Identifiers) FriendlyIdentifier() (out string) { + a, b := ii.Idents() + if len(b) > 0 { + return b[0] + } + + if len(a) > 0 { + return strconv.FormatUint(a[0], 10) + } + + return +} diff --git a/server/pkg/envoyx/node_test.go b/server/pkg/envoyx/node_test.go new file mode 100644 index 000000000..7b3bb38eb --- /dev/null +++ b/server/pkg/envoyx/node_test.go @@ -0,0 +1,118 @@ +package envoyx + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNodeForRef(t *testing.T) { + req := require.New(t) + + ref := Ref{ + ResourceType: "a", + Identifiers: MakeIdentifiers("a1", "a2"), + Scope: Scope{ + ResourceType: "a", + Identifiers: MakeIdentifiers("a1", "a2"), + }, + } + + t.Run("no nodes defined", func(t *testing.T) { + req.Nil(NodeForRef(ref)) + }) + + t.Run("not found", func(t *testing.T) { + nn := NodeForRef(ref, NodeSet{{ + // The type misses + ResourceType: "b", + Identifiers: MakeIdentifiers("a1", "a2"), + Scope: Scope{ + ResourceType: "a", + Identifiers: MakeIdentifiers("a1", "a2"), + }, + }}...) + req.Nil(nn) + }) + + t.Run("not found wrong scope", func(t *testing.T) { + nn := NodeForRef(ref, NodeSet{{ + // The type misses + ResourceType: "a", + Identifiers: MakeIdentifiers("a1", "a2"), + Scope: Scope{ + ResourceType: "b", + Identifiers: MakeIdentifiers("a1", "a2"), + }, + }}...) + req.Nil(nn) + }) + + t.Run("found", func(t *testing.T) { + nn := NodeForRef(ref, NodeSet{{ + // The type misses + ResourceType: "a", + Identifiers: MakeIdentifiers("a1", "a2"), + Scope: Scope{ + ResourceType: "a", + Identifiers: MakeIdentifiers("a1", "a2"), + }, + }}...) + req.NotNil(nn) + }) +} + +func TestIdents(t *testing.T) { + req := require.New(t) + + ii := MakeIdentifiers("asdf", "asd123", "321das", "12367831412") + + ints, ss := ii.Idents() + req.Len(ss, 3) + req.Contains(ss, "asdf") + req.Contains(ss, "asd123") + req.Contains(ss, "321das") + + req.Len(ints, 1) + req.Contains(ints, uint64(12367831412)) +} + +func TestIntersection(t *testing.T) { + req := require.New(t) + + aa := MakeIdentifiers("a", "b") + bb := MakeIdentifiers("a", "b") + cc := MakeIdentifiers("b", "c") + dd := MakeIdentifiers("c", "d") + + t.Run("complete overlap", func(t *testing.T) { + req.True(aa.HasIntersection(bb)) + }) + + t.Run("partial overlap", func(t *testing.T) { + req.True(aa.HasIntersection(cc)) + }) + + t.Run("completely off", func(t *testing.T) { + req.False(aa.HasIntersection(dd)) + }) +} + +func TestFriendlyIdentifier(t *testing.T) { + req := require.New(t) + + t.Run("empty", func(t *testing.T) { + ii := MakeIdentifiers() + req.Equal(ii.FriendlyIdentifier(), "") + }) + + t.Run("regular", func(t *testing.T) { + ii := MakeIdentifiers("h1", "h2", "123123123") + req.Equal(ii.FriendlyIdentifier(), "h1") + }) + + t.Run("fallback to ID", func(t *testing.T) { + ii := MakeIdentifiers("123123123") + req.Equal(ii.FriendlyIdentifier(), "123123123") + }) +} diff --git a/server/pkg/envoyx/store.go b/server/pkg/envoyx/store.go new file mode 100644 index 000000000..e83f01d4c --- /dev/null +++ b/server/pkg/envoyx/store.go @@ -0,0 +1,44 @@ +package envoyx + +import "context" + +func (svc *service) decodeStore(ctx context.Context, p DecodeParams) (nn NodeSet, err error) { + var aux NodeSet + for _, d := range svc.decoders[DecodeTypeStore] { + aux, err = d.Decode(ctx, p) + if err != nil { + return + } + nn = append(nn, aux...) + } + + return +} + +func (svc *service) encodeStore(ctx context.Context, dg *depGraph, p EncodeParams) (err error) { + // Prepping + // + // @note this is ok for now but if we add things like importing into + // multiple Cortezas at the same time, this won't be ok and each + // encoder should get it's own thing + for rt, nn := range depNodesByResourceType(dg.allNodes()...) { + for _, e := range svc.preparers[EncodeTypeStore] { + err = e.Prepare(ctx, p, rt, OmitPlaceholderNodes(unpackDepNodes(nn...)...)) + if err != nil { + return + } + } + } + + // Encoding + for rt, nn := range NodesByResourceType(dg.Roots()...) { + for _, se := range svc.encoders[EncodeTypeStore] { + err = se.Encode(ctx, p, rt, OmitPlaceholderNodes(nn...), dg) + if err != nil { + return + } + } + } + + return +}