diff --git a/pkg/envoy/builder.go b/pkg/envoy/builder.go index e2dc57ec7..4845f7601 100644 --- a/pkg/envoy/builder.go +++ b/pkg/envoy/builder.go @@ -37,7 +37,51 @@ func NewBuilder(pp ...Preparer) *builder { } } +// Build builds the graph that is used for structured data processing +// +// Outline: +// 1. Build an initial graph so that we can do some structured preprocessing. +// 2. Preprocess the resources based on the initial graph. The initial graph +// should remain unchanged. Preprocessing can request additional references and +// constraints. +// 3. Build a final graph based on the preprocessing modifications. func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph, error) { + var err error + + g := b.buildGraph(rr) + if err != nil { + return nil, err + } + + // Do any dep. related preprocessing + var state *ResourceState + err = func() error { + for { + state, err = g.NextInverted(ctx) + if err != nil { + return err + } else if state == nil { + return nil + } + + for _, p := range b.pp { + err = p.Prepare(ctx, state) + if err != nil { + return err + } + } + } + }() + + if err != nil { + return nil, err + } + + g = b.buildGraph(rr) + return g, nil +} + +func (b *builder) buildGraph(rr []resource.Interface) *graph { g := newGraph() // Prepare nodes for all resources @@ -46,9 +90,6 @@ func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph, nn = nn.add(newNode(r)) } - // Let's keep track of the missing deps. - mMap := make(map[resource.Interface]resource.RefSet) - // Index all resources for nicer lookups nIndex := make(nodeIndex) nIndex.Add(nn...) @@ -73,43 +114,7 @@ func (b *builder) Build(ctx context.Context, rr ...resource.Interface) (*graph, } g.addNode(cNode) - if len(missingRefs) > 0 { - mMap[cNode.res] = missingRefs - } } - // Do any dep. related preprocessing - var state *ResourceState - var err error - - err = func() error { - for { - state, err = g.NextInverted(ctx) - if err != nil { - return err - } else if state == nil { - return nil - } - - // Copy state so we don't alter the original one - nState := state - nState.MissingDeps = mMap[state.Res] - - for _, p := range b.pp { - err = p.Prepare(ctx, nState) - if err != nil { - return err - } - } - } - }() - - if err != nil { - return nil, err - } - - g.Relink() - g.reset() - - return g, nil + return g } diff --git a/pkg/envoy/graph.go b/pkg/envoy/graph.go index 82a3c3e3d..30ac6db31 100644 --- a/pkg/envoy/graph.go +++ b/pkg/envoy/graph.go @@ -24,7 +24,6 @@ type ( ResourceState struct { Res resource.Interface - MissingDeps resource.RefSet Conflicting bool DepResources []resource.Interface ParentResources []resource.Interface @@ -88,29 +87,6 @@ func (g *graph) markConflicting(n *node) { g.conflicting = g.conflicting.add(n) } -func (g *graph) Relink() { - for _, n := range g.nn { - n.cc = make(nodeSet, 0, len(n.cc)) - n.pp = make(nodeSet, 0, len(n.pp)) - } - - for res := range g.resIndex { - n := g.resIndex[res] - if n == nil { - return - } - - for _, ref := range res.Refs() { - // else find the node and link to it (if we can) - m := g.nn.findByRef(ref) - if m != nil { - g.addChild(n, m) - g.addParent(m, n) - } - } - } -} - func (g *graph) NextInverted(ctx context.Context) (s *ResourceState, err error) { g.inverted = true defer func() { diff --git a/pkg/envoy/node.go b/pkg/envoy/node.go index a746494dc..d7675ca26 100644 --- a/pkg/envoy/node.go +++ b/pkg/envoy/node.go @@ -12,7 +12,10 @@ type ( cc nodeSet } - nodeIndex map[string]map[string]*node + // resource type -> identifier -> []nodes + // There can be multiple resources with same identifier; for example + // two modules under different namespaces. + nodeIndex map[string]map[string]nodeSet ) func newNode(res resource.Interface) *node { @@ -37,18 +40,6 @@ func (nn nodeSet) filter(f func(n *node) bool) nodeSet { return mm } -func (nn nodeSet) findByRef(ref *resource.Ref) *node { - for _, n := range nn { - if n.res.ResourceType() == ref.ResourceType { - if n.res.Identifiers().HasAny(ref.Identifiers) { - return n - } - } - } - - return nil -} - func (nn nodeSet) has(m *node) bool { for _, n := range nn { if n == m { @@ -81,26 +72,47 @@ func (ri nodeIndex) Add(nn ...*node) { for _, n := range nn { rt := n.res.ResourceType() if _, has := ri[rt]; !has { - ri[rt] = make(map[string]*node) + ri[rt] = make(map[string]nodeSet) } for i := range n.res.Identifiers() { - ri[rt][i] = n + if ri[rt][i] == nil { + ri[rt][i] = make(nodeSet, 0, 5) + } + ri[rt][i] = append(ri[rt][i], n) } } } func (ri nodeIndex) GetRef(ref *resource.Ref) *node { - res, has := ri[ref.ResourceType] + refIi, has := ri[ref.ResourceType] if !has { return nil } for i := range ref.Identifiers { - r, has := res[i] - if has { - return r + rr, has := refIi[i] + if !has || len(rr) == 0 { + continue } + + // No constraints? no worries + if ref.Constraints == nil || len(ref.Constraints) == 0 { + return rr[0] + } + + // Constraints? check if ok + // If this loop makes you sick, don't worry; this will be at most 3x3x1 + for _, r := range rr { + for _, c := range ref.Constraints { + for _, ref := range r.res.Refs() { + if ref.ResourceType == c.ResourceType && ref.Identifiers.HasAny(c.Identifiers) { + return r + } + } + } + } + } return nil diff --git a/pkg/envoy/resource/compose_chart.go b/pkg/envoy/resource/compose_chart.go index 317ee0abc..7447ff275 100644 --- a/pkg/envoy/resource/compose_chart.go +++ b/pkg/envoy/resource/compose_chart.go @@ -28,7 +28,7 @@ func NewComposeChart(res *types.Chart, nsRef string, mmRef []string) *ComposeCha r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) for i, mRef := range mmRef { - r.ModRef[i] = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, mRef) + r.ModRef[i] = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, mRef).Constraint(r.NsRef) } return r diff --git a/pkg/envoy/resource/compose_module.go b/pkg/envoy/resource/compose_module.go index 5458d214e..fead72a85 100644 --- a/pkg/envoy/resource/compose_module.go +++ b/pkg/envoy/resource/compose_module.go @@ -33,7 +33,7 @@ func NewComposeModule(res *types.Module, nsRef string) *ComposeModule { case "Record": refM := f.Options.String("module") if refM != "" && refM != "0" { - r.ModRef = append(r.ModRef, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, refM)) + r.ModRef = append(r.ModRef, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, refM).Constraint(r.NsRef)) } } } diff --git a/pkg/envoy/resource/compose_page.go b/pkg/envoy/resource/compose_page.go index 0040a0049..14591ee49 100644 --- a/pkg/envoy/resource/compose_page.go +++ b/pkg/envoy/resource/compose_page.go @@ -32,11 +32,11 @@ func NewComposePage(pg *types.Page, nsRef, modRef, parentRef string) *ComposePag r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) if modRef != "" { - r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef) + r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef).Constraint(r.NsRef) } if parentRef != "" { - r.ParentRef = r.AddRef(COMPOSE_PAGE_RESOURCE_TYPE, parentRef) + r.ParentRef = r.AddRef(COMPOSE_PAGE_RESOURCE_TYPE, parentRef).Constraint(r.NsRef) } for _, b := range pg.Blocks { @@ -44,13 +44,13 @@ func NewComposePage(pg *types.Page, nsRef, modRef, parentRef string) *ComposePag case "RecordList": id, _ := b.Options["module"].(string) if id != "" { - r.ModRefs = append(r.ModRefs, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, id)) + r.ModRefs = append(r.ModRefs, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, id).Constraint(r.NsRef)) } case "Chart": id, _ := b.Options["chart"].(string) if id != "" { - r.ChartRefs = append(r.ChartRefs, r.AddRef(COMPOSE_CHART_RESOURCE_TYPE, id)) + r.ChartRefs = append(r.ChartRefs, r.AddRef(COMPOSE_CHART_RESOURCE_TYPE, id).Constraint(r.NsRef)) } case "Calendar": @@ -60,7 +60,7 @@ func NewComposePage(pg *types.Page, nsRef, modRef, parentRef string) *ComposePag fOpts, _ := (feed["options"]).(map[string]interface{}) id, _ := fOpts["module"].(string) if id != "" { - r.ModRefs = append(r.ModRefs, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, id)) + r.ModRefs = append(r.ModRefs, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, id).Constraint(r.NsRef)) } } @@ -70,7 +70,7 @@ func NewComposePage(pg *types.Page, nsRef, modRef, parentRef string) *ComposePag mops, _ := m.(map[string]interface{}) id, _ := mops["module"].(string) if id != "" { - r.ModRefs = append(r.ModRefs, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, id)) + r.ModRefs = append(r.ModRefs, r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, id).Constraint(r.NsRef)) } } } diff --git a/pkg/envoy/resource/compose_record.go b/pkg/envoy/resource/compose_record.go index 6b79f4be8..8e9c0e9fe 100644 --- a/pkg/envoy/resource/compose_record.go +++ b/pkg/envoy/resource/compose_record.go @@ -52,7 +52,7 @@ func NewComposeRecordSet(w crsWalker, nsRef, modRef string) *ComposeRecord { r.AddIdentifier(identifiers(modRef)...) r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) - r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef) + r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef).Constraint(r.NsRef) return r } diff --git a/pkg/envoy/resource/compose_record_template.go b/pkg/envoy/resource/compose_record_template.go index 5945a5912..8324c71f4 100644 --- a/pkg/envoy/resource/compose_record_template.go +++ b/pkg/envoy/resource/compose_record_template.go @@ -41,8 +41,8 @@ func NewComposeRecordTemplate(modRef, nsRef, name string, fieldMap MappingTplSet r.Key = key r.FieldMap = fieldMap - r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef) r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) + r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef).Constraint(r.NsRef) r.SetResourceType(DATA_SOURCE_RESOURCE_TYPE) r.AddIdentifier(identifiers(name)...) diff --git a/pkg/envoy/resource/types.go b/pkg/envoy/resource/types.go index c62fdece0..3e8a9cfdb 100644 --- a/pkg/envoy/resource/types.go +++ b/pkg/envoy/resource/types.go @@ -26,6 +26,7 @@ type ( // @todo should this become node type instead? ResourceType string Identifiers Identifiers + Constraints RefSet } Identifiers map[string]bool @@ -125,3 +126,17 @@ func (rr InterfaceSet) Walk(f func(r Interface) error) (err error) { return nil } + +// Constraint returns the current reference with added constraint +func (r *Ref) Constraint(c *Ref) *Ref { + if r.Constraints == nil { + r.Constraints = make(RefSet, 0, 1) + } + + r.Constraints = append(r.Constraints, &Ref{ + ResourceType: c.ResourceType, + Identifiers: MakeIdentifiers(c.Identifiers.StringSlice()...), + }) + + return r +} diff --git a/pkg/envoy/store/compose_record.go b/pkg/envoy/store/compose_record.go index 42e370336..8bf5c83b5 100644 --- a/pkg/envoy/store/compose_record.go +++ b/pkg/envoy/store/compose_record.go @@ -72,7 +72,7 @@ func (n *composeRecordState) Prepare(ctx context.Context, s store.Storer, state refM := f.Options.String("module") if refM != "" && refM != "0" { // Make a reference with that module's records - n.res.AddRef(resource.COMPOSE_RECORD_RESOURCE_TYPE, refM) + n.res.AddRef(resource.COMPOSE_RECORD_RESOURCE_TYPE, refM).Constraint(n.res.NsRef) } } }