Cleanup Envoy

* Improve conflict resolution algorithm.
* Tweak resource merge algorithms.
* Bump test coverage.

pkg/envoy/builder.go

@@ -49,9 +49,6 @@ func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph,
 	var err error
 
 	g := b.buildGraph(rr)
-	if err != nil {
-		return nil, err
-	}
 
 	// Do any dep. related preprocessing
 	var state *ResourceState

pkg/envoy/graph.go

@@ -16,7 +16,6 @@ type (
 
 		// Config flags
 		inverted bool
-		dry      bool
 
 		processed   nodeMap
 		conflicting nodeMap
@@ -28,11 +27,6 @@ type (
 		DepResources    []resource.Interface
 		ParentResources []resource.Interface
 	}
-
-	// Provides a filter for graph iterator
-	iterFilter struct {
-		resourceType string
-	}
 )
 
 func newGraph() *graph {
@@ -54,6 +48,10 @@ func (g *graph) addNode(nn ...*node) {
 
 func (g *graph) removeNode(nn ...*node) {
 	g.nn = g.nn.remove(nn...)
+	for _, n := range nn {
+		g.removeChild(n)
+		g.removeParent(n)
+	}
 }
 
 func (g *graph) invert() {
@@ -93,32 +91,12 @@ func (g *graph) NextInverted(ctx context.Context) (s *ResourceState, err error)
 		g.inverted = false
 	}()
 
-	return g.next(ctx, nil)
+	return g.Next(ctx)
 }
 
 func (g *graph) Next(ctx context.Context) (s *ResourceState, err error) {
-	return g.next(ctx, nil)
-}
-
-func (g *graph) NextOf(ctx context.Context, resTrype string) (s *ResourceState, err error) {
-	return g.next(ctx, &iterFilter{resourceType: resTrype})
-}
-
-func (g *graph) next(ctx context.Context, f *iterFilter) (s *ResourceState, err error) {
 	upNN := g.removeProcessed(g.nodes())
 
-	if f != nil {
-		upC := make(nodeSet, 0, len(upNN))
-		if f.resourceType != "" {
-			for _, n := range upNN {
-				if n.res.ResourceType() == f.resourceType {
-					upC = append(upC, n)
-				}
-			}
-		}
-		upNN = upC
-	}
-
 	// We are done here
 	if len(upNN) <= 0 {
 		return nil, nil
@@ -139,36 +117,76 @@ func (g *graph) next(ctx context.Context, f *iterFilter) (s *ResourceState, err
 		// Prepare the required context for the processing.
 		// Perform some basic cleanup.
 		es := g.prepExecState(nx, false)
-
 		g.markProcessed(nx)
 
 		return es, nil
 	}
 
-	// There are only conflicting nodes...
-	// Determine a conflicting node that should be partially resolved
-	for _, upN := range upNN {
-		if !g.conflicting.has(upN) {
-			nx = upN
-			break
+	// There are only conflicting nodes
+	// Try to get a cycle node to resolve the conflict
+	nx = g.findCycleNode(g.removeConflicting(upNN))
+
+	if nx == nil {
+		// This is basically impossible, unless I've messed up the algorithm
+		return nil, errors.New("could not determine non-conflicting node")
+	}
+
+	// Prepare the required context for the processing.
+	es := g.prepExecState(nx, true)
+	g.markConflicting(nx)
+
+	return es, nil
+}
+
+// findCycleNode returns the first graph node that caused a cycle
+//
+// General outline:
+//  * DFS from a start node(s)
+//  * if a child node is already in path, return that node
+//  * else return nil and cleanup the path until the first node with
+//    unprocessed child nodes
+//
+// @note we could complicate this further by doing cycle enumeration algorithms.
+//       I might do it when no one is watching :)
+func (g *graph) findCycleNode(nn nodeSet) *node {
+	path := make(nodeMap)
+	processed := make(nodeMap)
+
+	for _, n := range nn {
+		if !processed.has(n) {
+			m := g.traverse(n, path, processed)
+			if m != nil {
+				return m
+			}
 		}
 	}
 
-	if nx != nil {
-		// Prepare the required context for the processing.
-		es := g.prepExecState(nx, true)
-
-		g.markConflicting(nx)
-
-		return es, nil
-	} else {
-		return nil, errors.New("Uhoh, couldn't determine node; @todo error")
-	}
+	return nil
 }
 
-func (g *graph) reset() {
-	g.conflicting = make(nodeMap)
-	g.processed = make(nodeMap)
+func (g *graph) traverse(n *node, path, processed nodeMap) *node {
+	processed.add(n)
+
+	// Found ourselves a cycle
+	if path.has(n) {
+		return n
+	}
+	// Nothing else to look at
+	if len(g.childNodes(n)) == 0 {
+		return nil
+	}
+
+	path.add(n)
+
+	for _, c := range g.childNodes(n) {
+		m := g.traverse(c, path, processed)
+		if m != nil {
+			return m
+		}
+	}
+
+	path.remove(n)
+	return nil
 }
 
 // util
@@ -206,11 +224,6 @@ func (g *graph) addChild(n *node, mm ...*node) {
 }
 
 func (g *graph) removeChild(n *node, mm ...*node) {
-	// Dryrun shouldn't remove any nodes
-	if g.dry {
-		return
-	}
-
 	if len(mm) <= 0 {
 		if g.inverted {
 			n.pp = make(nodeSet, 0, 10)
@@ -235,11 +248,6 @@ func (g *graph) addParent(n *node, mm ...*node) {
 }
 
 func (g *graph) removeParent(n *node, mm ...*node) {
-	// Dryrun shouldn't remove any nodes
-	if g.dry {
-		return
-	}
-
 	if len(mm) <= 0 {
 		if g.inverted {
 			n.cc = make(nodeSet, 0, 10)

pkg/envoy/graph_test.go

@@ -108,3 +108,322 @@ func TestGraph_Walk(t *testing.T) {
 		req.False(s.Conflicting)
 	})
 }
+
+func TestGraph_WalkFlags(t *testing.T) {
+	req := require.New(t)
+	ctx := context.Background()
+
+	t.Run("simple node link; a -> b -> c -> c", func(t *testing.T) {
+		bl := NewBuilder()
+
+		a := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id1": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id2": true}}},
+		}
+
+		b := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id2": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id3": true}}},
+		}
+		c := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id3": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id3": true}}},
+		}
+
+		rr := []resource.Interface{
+			a,
+			b,
+			c,
+		}
+
+		g, err := bl.Build(ctx, rr...)
+		req.NoError(err)
+
+		s, err := g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(c, s.Res)
+		req.True(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(b, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(a, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(c, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Nil(s)
+	})
+
+	t.Run("simple node link; a -> b -> c -> d -> b; a -> b -> e", func(t *testing.T) {
+		bl := NewBuilder()
+
+		a := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id1": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id2": true}}},
+		}
+
+		b := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id2": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id3": true}}, &resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id5": true}}},
+		}
+		c := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id3": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id4": true}}},
+		}
+		d := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id4": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id2": true}}},
+		}
+		e := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id5": true},
+			refs:        nil,
+		}
+
+		rr := []resource.Interface{
+			a,
+			b,
+			c,
+			d,
+			e,
+		}
+
+		g, err := bl.Build(ctx, rr...)
+		req.NoError(err)
+
+		s, err := g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(e, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(b, s.Res)
+		req.True(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(a, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(d, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(c, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(b, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Nil(s)
+	})
+
+	t.Run("simple node link; a -> b -> c -> d -> b; a -> b -> c -> e -> b", func(t *testing.T) {
+		bl := NewBuilder()
+
+		a := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id1": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id2": true}}},
+		}
+
+		b := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id2": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id3": true}}},
+		}
+		c := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id3": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id4": true}}, &resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id5": true}}},
+		}
+		d := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id4": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id2": true}}},
+		}
+		e := &testResource{
+			resType:     "test:resource:1:",
+			identifiers: resource.Identifiers{"id5": true},
+			refs:        resource.RefSet{&resource.Ref{ResourceType: "test:resource:1:", Identifiers: resource.Identifiers{"id2": true}}},
+		}
+
+		rr := []resource.Interface{
+			a,
+			b,
+			c,
+			d,
+			e,
+		}
+
+		g, err := bl.Build(ctx, rr...)
+		req.NoError(err)
+
+		s, err := g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(b, s.Res)
+		req.True(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(a, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(d, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(e, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(c, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Equal(b, s.Res)
+		req.False(s.Conflicting)
+
+		s, err = g.NextInverted(ctx)
+		req.NoError(err)
+		req.Nil(s)
+	})
+}
+
+func TestGraph_Utilities(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("adding & removing nodes", func(t *testing.T) {
+		g := newGraph()
+		n1 := newNode(nil)
+		n2 := newNode(nil)
+
+		// Adding nodes
+		g.addNode(n1, n2)
+		req.Len(g.nn, 2)
+		req.Equal(n1, g.nn[0])
+		req.Equal(n2, g.nn[1])
+
+		// Dependencies
+		g.addChild(n1, n2)
+		g.addParent(n2, n1)
+		req.True(g.childNodes(n1).has(n2))
+		req.True(g.parentNodes(n2).has(n1))
+
+		// Removing nodes
+		g.removeNode(n1, n2)
+		req.False(g.childNodes(n1).has(n2))
+		req.False(g.parentNodes(n2).has(n1))
+	})
+
+	t.Run("inverted graph", func(t *testing.T) {
+		g := newGraph()
+		n1 := newNode(nil)
+		n2 := newNode(nil)
+
+		// Adding nodes
+		g.addNode(n1, n2)
+		req.Len(g.nn, 2)
+		req.Equal(n1, g.nn[0])
+		req.Equal(n2, g.nn[1])
+
+		// Dependencies
+		g.addChild(n1, n2)
+		g.addParent(n2, n1)
+		g.invert()
+		req.True(g.parentNodes(n1).has(n2))
+		req.True(g.childNodes(n2).has(n1))
+	})
+
+	t.Run("inverted child dependencies", func(t *testing.T) {
+		g := newGraph()
+		n1 := newNode(nil)
+		n2 := newNode(nil)
+		n3 := newNode(nil)
+
+		g.addNode(n1, n2, n3)
+
+		g.addChild(n1, n2)
+		g.invert()
+		g.addChild(n1, n3)
+		req.Equal(n2, n1.cc[0])
+		req.Equal(n3, n1.pp[0])
+
+		g.removeChild(n1, n3)
+		req.Len(n1.pp, 0)
+	})
+
+	t.Run("inverted parent dependencies", func(t *testing.T) {
+		g := newGraph()
+		n1 := newNode(nil)
+		n2 := newNode(nil)
+		n3 := newNode(nil)
+
+		g.addNode(n1, n2, n3)
+
+		g.addParent(n1, n2)
+		g.invert()
+		g.addParent(n1, n3)
+		req.Equal(n2, n1.pp[0])
+		req.Equal(n3, n1.cc[0])
+
+		g.removeParent(n1, n3)
+		req.Len(n1.cc, 0)
+	})
+
+	// This one is just for codecoverage; the logic is nothing
+	t.Run("remove all child/parent nodes", func(t *testing.T) {
+		g := newGraph()
+		n1 := newNode(nil)
+		n2 := newNode(nil)
+
+		g.addNode(n1, n2)
+		g.addChild(n1, n2)
+		g.addParent(n1, n2)
+
+		g.removeChild(n1)
+		g.removeParent(n1)
+		req.Len(n1.cc, 0)
+		req.Len(n1.pp, 0)
+
+		// Inverted graph
+		g.addChild(n1, n2)
+		g.addParent(n1, n2)
+		g.invert()
+
+		g.removeChild(n1)
+		g.removeParent(n1)
+		req.Len(n1.cc, 0)
+		req.Len(n1.pp, 0)
+	})
+}

pkg/envoy/marshaler.go

@@ -29,7 +29,6 @@ func CollectNodes(ii ...interface{}) (nn []resource.Interface, err error) {
 
 		case Marshaller:
 			if tmp, err := c.MarshalEnvoy(); err != nil {
-				println(err)
 				return nil, err
 			} else {
 				nn = append(nn, tmp...)

pkg/envoy/node.go

@@ -30,16 +30,6 @@ func (nn nodeSet) add(mm ...*node) nodeSet {
 	return append(nn, mm...)
 }
 
-func (nn nodeSet) filter(f func(n *node) bool) nodeSet {
-	mm := make(nodeSet, 0, len(nn))
-	for _, n := range nn {
-		if f(n) {
-			mm = append(mm, n)
-		}
-	}
-	return mm
-}
-
 func (nn nodeSet) has(m *node) bool {
 	for _, n := range nn {
 		if n == m {

pkg/envoy/resource/types.go

@@ -64,14 +64,6 @@ func (ri Identifiers) Add(ii ...string) Identifiers {
 	return ri
 }
 
-func (ri Identifiers) Remove(ii ...string) Identifiers {
-	for _, i := range ii {
-		delete(ri, i)
-	}
-
-	return ri
-}
-
 func (ri Identifiers) HasAny(ii Identifiers) bool {
 	for i := range ii {
 		if ri[i] {
@@ -98,26 +90,6 @@ func (ri Identifiers) First() string {
 	return ss[0]
 }
 
-func (ss RefSet) FilterByResourceType(rt string) RefSet {
-	rr := make(RefSet, 0, len(ss))
-
-	for _, s := range ss {
-		if s.ResourceType == rt {
-			rr = append(rr, s)
-		}
-	}
-
-	return rr
-}
-
-func (ss RefSet) FirstByResourceType(rt string) *Ref {
-	rr := ss.FilterByResourceType(rt)
-	if len(rt) <= 0 {
-		return nil
-	}
-	return rr[0]
-}
-
 func (rr InterfaceSet) Walk(f func(r Interface) error) (err error) {
 	for _, r := range rr {
 		err = f(r)

pkg/envoy/store/application.go

@@ -123,6 +123,17 @@ func mergeApplications(a, b *types.Application) *types.Application {
 	if c.Name == "" {
 		c.Name = b.Name
 	}
+	c.OwnerID = b.OwnerID
+
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
 
 	// I'll just compare the entire struct for now
 	if c.Unify == nil || *c.Unify == (types.ApplicationUnify{}) {

pkg/envoy/store/application_test.go

@@ -0,0 +1,54 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/system/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestApplication_Merger(t *testing.T) {
+	req := require.New(t)
+
+	ub := &types.ApplicationUnify{
+		Config: "config",
+		Icon:   "icon",
+	}
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Application{}
+	full := &types.Application{
+		Name:      "name",
+		OwnerID:   1,
+		Enabled:   true,
+		Unify:     ub,
+		CreatedAt: now,
+		UpdatedAt: nowP,
+		DeletedAt: nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeApplications(empty, full)
+		req.Equal("name", c.Name)
+		req.Equal(uint64(1), c.OwnerID)
+		req.False(c.Enabled)
+		req.Equal(ub, c.Unify)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeApplications(full, empty)
+		req.Equal("name", c.Name)
+		req.Equal(uint64(0), c.OwnerID)
+		req.True(c.Enabled)
+		req.Equal(ub, c.Unify)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/compose_chart.go

@@ -173,6 +173,16 @@ func mergeComposeChart(a, b *types.Chart) *types.Chart {
 	if c.Name == "" {
 		c.Name = b.Name
 	}
+	c.NamespaceID = b.NamespaceID
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
 
 	if len(c.Config.Reports) <= 0 {
 		c.Config.Reports = b.Config.Reports

pkg/envoy/store/compose_chart_test.go

@@ -0,0 +1,54 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComposeChart_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Chart{}
+	full := &types.Chart{
+		Handle: "handle",
+		Name:   "name",
+		Config: types.ChartConfig{
+			Reports:     []*types.ChartConfigReport{&types.ChartConfigReport{}},
+			ColorScheme: "colorScheme",
+		},
+		NamespaceID: 1,
+		CreatedAt:   now,
+		UpdatedAt:   nowP,
+		DeletedAt:   nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeComposeChart(empty, full)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.Len(c.Config.Reports, 1)
+		req.Equal("colorScheme", c.Config.ColorScheme)
+		req.Equal(uint64(1), c.NamespaceID)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeComposeChart(full, empty)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.Len(c.Config.Reports, 1)
+		req.Equal("colorScheme", c.Config.ColorScheme)
+		req.Equal(uint64(0), c.NamespaceID)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/compose_module.go

@@ -245,6 +245,8 @@ func mergeComposeModuleFields(a, b types.ModuleFieldSet) types.ModuleFieldSet {
 				continue
 			}
 
+			fa.ModuleID = fb.ModuleID
+			fa.Place = fb.Place
 			if fa.Kind == "" {
 				fa.Kind = fb.Kind
 			}
@@ -257,9 +259,31 @@ func mergeComposeModuleFields(a, b types.ModuleFieldSet) types.ModuleFieldSet {
 			if fa.Options == nil {
 				fa.Options = fb.Options
 			}
-			if fa.DefaultValue == nil || len(fa.DefaultValue) <= 0 {
+			if fa.DefaultValue == nil {
 				fa.DefaultValue = fb.DefaultValue
 			}
+			if fa.Expressions.ValueExpr == "" {
+				fa.Expressions.ValueExpr = fb.Expressions.ValueExpr
+			}
+			if len(fa.Expressions.Sanitizers) == 0 {
+				fa.Expressions.Sanitizers = fb.Expressions.Sanitizers
+			}
+			if len(fa.Expressions.Validators) == 0 {
+				fa.Expressions.Validators = fb.Expressions.Validators
+			}
+			if len(fa.Expressions.Formatters) == 0 {
+				fa.Expressions.Formatters = fb.Expressions.Formatters
+			}
+
+			if fa.CreatedAt.IsZero() {
+				fa.CreatedAt = fb.CreatedAt
+			}
+			if fa.UpdatedAt == nil {
+				fa.UpdatedAt = fb.UpdatedAt
+			}
+			if fa.DeletedAt == nil {
+				fa.DeletedAt = fb.DeletedAt
+			}
 
 			goto out
 		}
@@ -282,6 +306,17 @@ func mergeComposeModule(a, b *types.Module) *types.Module {
 	if c.Name == "" {
 		c.Name = b.Name
 	}
+	c.NamespaceID = b.NamespaceID
+
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
 
 	// I'll just compare the entire struct for now
 	if c.Meta == nil {

pkg/envoy/store/compose_module_test.go

@@ -0,0 +1,111 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+	st "github.com/jmoiron/sqlx/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComposeModule_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Module{}
+	full := &types.Module{
+		Handle:      "handle",
+		Name:        "name",
+		Meta:        st.JSONText{},
+		NamespaceID: 1,
+
+		CreatedAt: now,
+		UpdatedAt: nowP,
+		DeletedAt: nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeComposeModule(empty, full)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.NotNil(c.Meta)
+		req.Equal(uint64(1), c.NamespaceID)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeComposeModule(full, empty)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.NotNil(c.Meta)
+		req.Equal(uint64(0), c.NamespaceID)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}
+
+func TestComposeModuleField_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := types.ModuleFieldSet{}
+	full := types.ModuleFieldSet{
+		&types.ModuleField{
+			ModuleID:     1,
+			Place:        2,
+			Kind:         "kind",
+			Name:         "name",
+			Label:        "label",
+			Options:      types.ModuleFieldOptions{},
+			DefaultValue: types.RecordValueSet{},
+			Expressions:  types.ModuleFieldExpr{},
+			CreatedAt:    now,
+			UpdatedAt:    nowP,
+			DeletedAt:    nowP,
+		},
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		cc := mergeComposeModuleFields(empty, full)
+		req.Len(cc, 1)
+		c := cc[0]
+
+		req.Equal(uint64(1), c.ModuleID)
+		req.Equal(2, c.Place)
+		req.Equal("kind", c.Kind)
+		req.Equal("name", c.Name)
+		req.Equal("label", c.Label)
+		req.NotNil(c.Options)
+		req.NotNil(c.DefaultValue)
+		req.NotNil(c.Expressions)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		cc := mergeComposeModuleFields(full, empty)
+		req.Len(cc, 1)
+		c := cc[0]
+
+		req.Equal(uint64(1), c.ModuleID)
+		req.Equal(2, c.Place)
+		req.Equal("kind", c.Kind)
+		req.Equal("name", c.Name)
+		req.Equal("label", c.Label)
+		req.NotNil(c.Options)
+		req.NotNil(c.DefaultValue)
+		req.NotNil(c.Expressions)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/compose_namespace.go

@@ -126,6 +126,16 @@ func mergeComposeNamespaces(a, b *types.Namespace) *types.Namespace {
 		c.Slug = b.Slug
 	}
 
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
+
 	// I'll just compare the entire struct for now
 	if c.Meta == (types.NamespaceMeta{}) {
 		c.Meta = b.Meta

pkg/envoy/store/compose_namespace_test.go

@@ -0,0 +1,53 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComposeNamespace_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Namespace{}
+	full := &types.Namespace{
+		Name:    "name",
+		Slug:    "slug",
+		Enabled: false,
+		Meta: types.NamespaceMeta{
+			Description: "dsc",
+			Subtitle:    "sub",
+		},
+
+		CreatedAt: now,
+		UpdatedAt: nowP,
+		DeletedAt: nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeComposeNamespaces(empty, full)
+		req.Equal("name", c.Name)
+		req.Equal("slug", c.Slug)
+		req.Equal("dsc", c.Meta.Description)
+		req.Equal("sub", c.Meta.Subtitle)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeComposeNamespaces(full, empty)
+		req.Equal("name", c.Name)
+		req.Equal("slug", c.Slug)
+		req.Equal("dsc", c.Meta.Description)
+		req.Equal("sub", c.Meta.Subtitle)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/compose_page.go

@@ -298,9 +298,10 @@ func (n *composePageState) Encode(ctx context.Context, s store.Storer, state *en
 func mergeComposePage(a, b *types.Page) *types.Page {
 	c := a.Clone()
 
-	if c.SelfID <= 0 {
-		c.SelfID = b.SelfID
-	}
+	c.SelfID = b.SelfID
+	c.NamespaceID = b.NamespaceID
+	c.ModuleID = b.ModuleID
+	c.Weight = b.Weight
 	if c.Handle == "" {
 		c.Handle = b.Handle
 	}
@@ -317,6 +318,16 @@ func mergeComposePage(a, b *types.Page) *types.Page {
 		c.Children = b.Children
 	}
 
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
+
 	return c
 }
 

pkg/envoy/store/compose_page_test.go

@@ -0,0 +1,69 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComposePage_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Page{}
+	full := &types.Page{
+		SelfID:      1,
+		NamespaceID: 2,
+		ModuleID:    3,
+		Handle:      "handle",
+		Title:       "title",
+		Description: "description",
+		Blocks:      types.PageBlocks{types.PageBlock{}},
+		Children:    types.PageSet{&types.Page{}},
+		Weight:      4,
+
+		CreatedAt: now,
+		UpdatedAt: nowP,
+		DeletedAt: nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeComposePage(empty, full)
+		req.Equal(uint64(1), c.SelfID)
+		req.Equal(uint64(2), c.NamespaceID)
+		req.Equal(uint64(3), c.ModuleID)
+
+		req.Equal("handle", c.Handle)
+		req.Equal("title", c.Title)
+		req.Equal("description", c.Description)
+		req.Len(c.Blocks, 1)
+		req.Len(c.Children, 1)
+		req.Equal(4, c.Weight)
+
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeComposePage(full, empty)
+		req.Equal(uint64(0), c.SelfID)
+		req.Equal(uint64(0), c.NamespaceID)
+		req.Equal(uint64(0), c.ModuleID)
+
+		req.Equal("handle", c.Handle)
+		req.Equal("title", c.Title)
+		req.Equal("description", c.Description)
+		req.Len(c.Blocks, 1)
+		req.Len(c.Children, 1)
+		req.Equal(0, c.Weight)
+
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/messaging_channel.go

@@ -149,6 +149,30 @@ func mergeMessagingChannels(a, b *types.Channel) *types.Channel {
 	if c.Name == "" {
 		c.Name = b.Name
 	}
+	if c.Topic == "" {
+		c.Topic = b.Topic
+	}
+	if c.Type == "" {
+		c.Type = b.Type
+	}
+	if c.Meta == nil {
+		c.Meta = b.Meta
+	}
+	c.MembershipPolicy = b.MembershipPolicy
+	c.CreatorID = b.CreatorID
+
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.ArchivedAt == nil {
+		c.ArchivedAt = b.ArchivedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
 
 	return &c
 }

pkg/envoy/store/messaging_channel_test.go

@@ -0,0 +1,62 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/messaging/types"
+	st "github.com/jmoiron/sqlx/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMessagingChannel_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Channel{}
+	full := &types.Channel{
+		Name:             "name",
+		Topic:            "topic",
+		Type:             types.ChannelTypeGroup,
+		Meta:             st.JSONText{},
+		MembershipPolicy: types.ChannelMembershipPolicyFeatured,
+		CreatorID:        1,
+
+		CreatedAt:  now,
+		UpdatedAt:  nowP,
+		ArchivedAt: nowP,
+		DeletedAt:  nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeMessagingChannels(empty, full)
+		req.Equal("name", c.Name)
+		req.Equal("topic", c.Topic)
+		req.Equal(types.ChannelTypeGroup, c.Type)
+		req.NotNil(c.Meta)
+		req.Equal(types.ChannelMembershipPolicyFeatured, c.MembershipPolicy)
+		req.Equal(uint64(1), c.CreatorID)
+
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.ArchivedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeMessagingChannels(full, empty)
+		req.Equal("name", c.Name)
+		req.Equal("topic", c.Topic)
+		req.Equal(types.ChannelTypeGroup, c.Type)
+		req.NotNil(c.Meta)
+		req.Equal(types.ChannelMembershipPolicyDefault, c.MembershipPolicy)
+		req.Equal(uint64(0), c.CreatorID)
+
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.ArchivedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/role.go

@@ -102,10 +102,10 @@ func (n *roleState) Encode(ctx context.Context, s store.Storer, state *envoy.Res
 		return nil
 
 	case resource.MergeLeft:
-		rl = mergeRole(n.rl, rl)
+		rl = mergeRoles(n.rl, rl)
 
 	case resource.MergeRight:
-		rl = mergeRole(rl, n.rl)
+		rl = mergeRoles(rl, n.rl)
 	}
 
 	err = store.UpdateRole(ctx, s, rl)
@@ -117,8 +117,8 @@ func (n *roleState) Encode(ctx context.Context, s store.Storer, state *envoy.Res
 	return nil
 }
 
-// mergeRole merges b into a, prioritising a
-func mergeRole(a, b *types.Role) *types.Role {
+// mergeRoles merges b into a, prioritising a
+func mergeRoles(a, b *types.Role) *types.Role {
 	c := *a
 
 	if c.Name == "" {
@@ -128,6 +128,19 @@ func mergeRole(a, b *types.Role) *types.Role {
 		c.Handle = b.Handle
 	}
 
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.ArchivedAt == nil {
+		c.ArchivedAt = b.ArchivedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
+
 	return &c
 }
 

pkg/envoy/store/role_test.go

@@ -0,0 +1,46 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/system/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRole_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.Role{}
+	full := &types.Role{
+		Name:       "name",
+		Handle:     "handle",
+		CreatedAt:  now,
+		UpdatedAt:  nowP,
+		ArchivedAt: nowP,
+		DeletedAt:  nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeRoles(empty, full)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.ArchivedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeRoles(full, empty)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.ArchivedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

pkg/envoy/store/user.go

@@ -138,10 +138,23 @@ func mergeUsers(a, b *types.User) *types.User {
 	if c.Kind == "" {
 		c.Kind = b.Kind
 	}
-	if *c.Meta == (types.UserMeta{}) {
+	if c.Meta == nil {
 		c.Meta = b.Meta
 	}
 
+	if c.CreatedAt.IsZero() {
+		c.CreatedAt = b.CreatedAt
+	}
+	if c.UpdatedAt == nil {
+		c.UpdatedAt = b.UpdatedAt
+	}
+	if c.SuspendedAt == nil {
+		c.SuspendedAt = b.SuspendedAt
+	}
+	if c.DeletedAt == nil {
+		c.DeletedAt = b.DeletedAt
+	}
+
 	return &c
 }
 

pkg/envoy/store/user_test.go

@@ -0,0 +1,58 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/system/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUser_Merger(t *testing.T) {
+	req := require.New(t)
+
+	now := time.Time{}
+	nowP := &time.Time{}
+
+	empty := &types.User{}
+	full := &types.User{
+		Username:    "username",
+		Email:       "email",
+		Name:        "name",
+		Handle:      "handle",
+		Kind:        types.BotUser,
+		Meta:        &types.UserMeta{},
+		CreatedAt:   now,
+		UpdatedAt:   nowP,
+		SuspendedAt: nowP,
+		DeletedAt:   nowP,
+	}
+
+	t.Run("merge on empty", func(t *testing.T) {
+		c := mergeUsers(empty, full)
+		req.Equal("username", c.Username)
+		req.Equal("email", c.Email)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.Equal(types.BotUser, c.Kind)
+		req.Equal(&types.UserMeta{}, c.Meta)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.SuspendedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+
+	t.Run("merge with empty", func(t *testing.T) {
+		c := mergeUsers(full, empty)
+		req.Equal("username", c.Username)
+		req.Equal("email", c.Email)
+		req.Equal("name", c.Name)
+		req.Equal("handle", c.Handle)
+		req.Equal(types.BotUser, c.Kind)
+		req.Equal(&types.UserMeta{}, c.Meta)
+		req.Equal(now, c.CreatedAt)
+		req.Equal(nowP, c.UpdatedAt)
+		req.Equal(nowP, c.SuspendedAt)
+		req.Equal(nowP, c.DeletedAt)
+	})
+}

tests/envoy/simple_test.go

@@ -643,6 +643,77 @@ func TestSimpleCases(t *testing.T) {
 				req.Equal("f1 value", rr2[0].Values.FilterByName("f1")[0].Value)
 			},
 		},
+
+		{
+			name:  "base access control",
+			suite: "simple",
+			file:  "access_control_base",
+			pre: func() (err error) {
+				return ce(
+					s.TruncateComposeRecords(ctx, nil),
+					s.TruncateComposeModuleFields(ctx),
+					s.TruncateComposeModules(ctx),
+					s.TruncateComposeNamespaces(ctx),
+					s.TruncateComposePages(ctx),
+					s.TruncateComposeCharts(ctx),
+					s.TruncateUsers(ctx),
+					s.TruncateRoles(ctx),
+					s.TruncateMessagingChannels(ctx),
+					s.TruncateRbacRules(ctx),
+				)
+			},
+			post: func(req *require.Assertions, err error) {
+				req.NoError(err)
+			},
+			check: func(req *require.Assertions) {
+
+				role, err := store.LookupRoleByHandle(ctx, s, "everyone")
+				req.NoError(err)
+				req.NotNil(role)
+
+				rr, _, err := store.SearchRbacRules(ctx, s, rbac.RuleFilter{})
+				req.NoError(err)
+				req.Len(rr, 18)
+
+				// Check that the role is ok
+				rr.Walk(func(r *rbac.Rule) error {
+					req.Equal(role.ID, r.RoleID)
+					return nil
+				})
+
+				resources := []string{
+					"compose:namespace:",
+					"compose:namespace:",
+					"compose:module:",
+					"compose:module:",
+					"compose:page:",
+					"compose:page:",
+					"compose:chart:",
+					"compose:chart:",
+					"messaging:channel:",
+					"messaging:channel:",
+					"system:role:",
+					"system:role:",
+					"system:user:",
+					"system:user:",
+					"system:application:",
+					"system:application:",
+					"compose",
+					"compose",
+				}
+
+				for i, res := range resources {
+					req.Equal(rbac.Resource(res), rr[i].Resource.TrimID())
+					if i%2 == 0 {
+						req.Equal(rbac.Operation("op1"), rr[i].Operation)
+						req.Equal(rbac.Allow, rr[i].Access)
+					} else {
+						req.Equal(rbac.Operation("op2"), rr[i].Operation)
+						req.Equal(rbac.Deny, rr[i].Access)
+					}
+				}
+			},
+		},
 	}
 
 	for _, c := range cases {

tests/envoy/testdata/simple/access_control_base.yaml

@@ -0,0 +1,92 @@
+allow:
+  everyone:
+    compose:
+      - op1
+deny:
+  everyone:
+    compose:
+      - op2
+
+namespaces:
+  ns1:
+    name: ns1 name
+
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+modules:
+  mod1:
+    name: mod1 name
+
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+pages:
+  pg1:
+    title: pg1 title
+
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+charts:
+  chr1:
+    name: chr1 name
+
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+applications:
+  - name: app1
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+channels:
+  - name: ch1
+    type: public
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+roles:
+  everyone:
+    name: everyone name
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2
+
+users:
+  u1:
+    name: u1 name
+    email: u1@example.tld
+    allow:
+      everyone:
+      - op1
+    deny:
+      everyone:
+      - op2