diff --git a/pkg/envoy/builder_test.go b/pkg/envoy/builder_test.go index 2297474d7..79737a6ce 100644 --- a/pkg/envoy/builder_test.go +++ b/pkg/envoy/builder_test.go @@ -13,6 +13,7 @@ type ( resType string identifiers resource.Identifiers refs resource.RefSet + ph bool } ) @@ -28,6 +29,14 @@ func (t *testResource) Refs() resource.RefSet { return t.refs } +func (t *testResource) MarkPlaceholder() { + t.ph = true +} + +func (t *testResource) Placeholder() bool { + return t.ph +} + func TestGraphBuilder_Rel(t *testing.T) { req := require.New(t) ctx := context.Background() diff --git a/pkg/envoy/resource/resource.go b/pkg/envoy/resource/resource.go index 9b929d817..dc423400b 100644 --- a/pkg/envoy/resource/resource.go +++ b/pkg/envoy/resource/resource.go @@ -13,6 +13,7 @@ type ( rt string ii Identifiers rr RefSet + ph bool ts *Timestamps us *Userstamps @@ -197,6 +198,20 @@ func (t *base) HasRefs() bool { return t.rr == nil || len(t.rr) == 0 } +// MarkPlaceholder denotes that the given resource should be treated as a placeholder +// +// Placeholder resources should not be encoded but should only provide additional +// context to resources that depend on it +func (t *base) MarkPlaceholder() { + t.ph = true +} + +// Placeholder resources should not be encoded but should only provide additional +// context to resources that depend on it +func (t *base) Placeholder() bool { + return t.ph +} + func IgnoreDepResolution(ref *Ref) bool { return ref.ResourceType == composeTypes.ModuleFieldResourceType } diff --git a/pkg/envoy/resource/types.go b/pkg/envoy/resource/types.go index d24560a59..9a9e30625 100644 --- a/pkg/envoy/resource/types.go +++ b/pkg/envoy/resource/types.go @@ -7,6 +7,9 @@ type ( Identifiers() Identifiers ResourceType() string Refs() RefSet + + MarkPlaceholder() + Placeholder() bool } InterfaceSet []Interface @@ -143,3 +146,33 @@ func (r *Ref) Constraint(c *Ref) *Ref { func (r *Ref) IsWildcard() bool { return r.Identifiers["*"] } + +// Unique returns only unique references +// +// Uniqueness is defined as "two references may not define +// the same resource type and identifier" combinations. +func (rr RefSet) Unique() RefSet { + out := make(RefSet, 0, len(rr)) + seen := make(map[string]Identifiers) + + for _, r := range rr { + ii, ok := seen[r.ResourceType] + + // type not seen at all, unique + if !ok { + out = append(out, r) + seen[r.ResourceType] = r.Identifiers + continue + } + + // not yet seen + if !ii.HasAny(r.Identifiers) { + out = append(out, r) + for i := range r.Identifiers { + seen[r.ResourceType][i] = true + } + } + } + + return out +} diff --git a/pkg/envoy/store/automation.go b/pkg/envoy/store/automation.go index 2b5349a59..e6c6cda18 100644 --- a/pkg/envoy/store/automation.go +++ b/pkg/envoy/store/automation.go @@ -6,6 +6,7 @@ import ( "github.com/cortezaproject/corteza-server/automation/types" "github.com/cortezaproject/corteza-server/pkg/envoy" + "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" ) @@ -144,6 +145,26 @@ func (df *DecodeFilter) automationFromResource(rr ...string) *DecodeFilter { return df } +func (df *DecodeFilter) automationFromRef(rr ...*resource.Ref) *DecodeFilter { + for _, r := range rr { + if strings.Index(r.ResourceType, "automation") < 0 { + continue + } + + switch r.ResourceType { + case types.WorkflowResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.AutomationWorkflows(&types.WorkflowFilter{ + Query: i, + Disabled: filter.StateInclusive, + }) + } + } + } + + return df +} + // AutomationWorkflows adds a new WorkflowFilter func (df *DecodeFilter) AutomationWorkflows(f *types.WorkflowFilter) *DecodeFilter { if df.automationWorkflow == nil { diff --git a/pkg/envoy/store/compose.go b/pkg/envoy/store/compose.go index facd73ea9..aef7202c7 100644 --- a/pkg/envoy/store/compose.go +++ b/pkg/envoy/store/compose.go @@ -450,6 +450,43 @@ func (df *DecodeFilter) composeFromResource(rr ...string) *DecodeFilter { return df } +func (df *DecodeFilter) composeFromRef(rr ...*resource.Ref) *DecodeFilter { + for _, r := range rr { + if strings.Index(r.ResourceType, "compose") < 0 { + continue + } + + switch r.ResourceType { + case types.NamespaceResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.ComposeNamespace(&types.NamespaceFilter{ + Query: i, + }) + } + case types.ModuleResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.ComposeModule(&types.ModuleFilter{ + Query: i, + }) + } + case types.PageResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.ComposePage(&types.PageFilter{ + Query: i, + }) + } + case types.ChartResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.ComposeChart(&types.ChartFilter{ + Query: i, + }) + } + } + } + + return df +} + // ComposeNamespace adds a new compose NamespaceFilter func (df *DecodeFilter) ComposeNamespace(f *types.NamespaceFilter) *DecodeFilter { if df.composeNamespace == nil { diff --git a/pkg/envoy/store/decoder.go b/pkg/envoy/store/decoder.go index c65cfdc7c..f9a1518d3 100644 --- a/pkg/envoy/store/decoder.go +++ b/pkg/envoy/store/decoder.go @@ -66,6 +66,14 @@ func (df *DecodeFilter) FromResource(rr ...string) *DecodeFilter { return df } +func (df *DecodeFilter) FromRef(rr ...*resource.Ref) *DecodeFilter { + df = df.systemFromRef(rr...) + df = df.automationFromRef(rr...) + df = df.composeFromRef(rr...) + + return df +} + func (aum auxMarshaller) MarshalEnvoy() ([]resource.Interface, error) { ii := make([]resource.Interface, 0, len(aum)) for _, m := range aum { diff --git a/pkg/envoy/store/encoder.go b/pkg/envoy/store/encoder.go index 70fd152a9..758bd6d11 100644 --- a/pkg/envoy/store/encoder.go +++ b/pkg/envoy/store/encoder.go @@ -105,6 +105,10 @@ func NewStoreEncoder(s store.Storer, cfg *EncoderConfig) envoy.PrepareEncoder { // It initializes and prepares the resource state for each provided resource func (se *storeEncoder) Prepare(ctx context.Context, ee ...*envoy.ResourceState) (err error) { f := func(rs resourceState, ers *envoy.ResourceState) error { + if rs == nil { + return nil + } + err = rs.Prepare(ctx, se.makePayload(ctx, se.s, ers)) if err != nil { return err @@ -115,6 +119,11 @@ func (se *storeEncoder) Prepare(ctx context.Context, ee ...*envoy.ResourceState) } for _, ers := range ee { + // Skip placeholders + if ers.Res.Placeholder() { + continue + } + switch res := ers.Res.(type) { // Compose resources case *resource.ComposeNamespace: @@ -142,7 +151,7 @@ func (se *storeEncoder) Prepare(ctx context.Context, ee ...*envoy.ResourceState) case *resource.RbacRule: err = f(newRbacRuleFromResource(res, se.cfg), ers) case *resource.ResourceTranslation: - // err = f(newResourceTranslationFromResource(res, se.cfg), ers) + err = f(newResourceTranslationFromResource(res, se.cfg), ers) // Automation resources case *resource.AutomationWorkflow: @@ -172,15 +181,22 @@ func (se *storeEncoder) Encode(ctx context.Context, p envoy.Provider) error { return nil } + // Skip placeholders + if ers.Res.Placeholder() { + continue + } + state := se.state[ers.Res] if state == nil { err = ErrResourceStateUndefined } else { - err = state.Encode(ctx, se.makePayload(ctx, s, ers)) + if state != nil { + err = state.Encode(ctx, se.makePayload(ctx, s, ers)) + } } if err != nil { - //return se.WrapError("encode", ers.Res, err) + return se.WrapError("encode", ers.Res, err) } } }) diff --git a/pkg/envoy/store/resource_translation_marshal.go b/pkg/envoy/store/resource_translation_marshal.go index 2e9e2b44a..f75244221 100644 --- a/pkg/envoy/store/resource_translation_marshal.go +++ b/pkg/envoy/store/resource_translation_marshal.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + automationTypes "github.com/cortezaproject/corteza-server/automation/types" composeTypes "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "github.com/cortezaproject/corteza-server/store" @@ -65,6 +66,12 @@ func (n *resourceTranslation) Prepare(ctx context.Context, pl *payload) (err err } func (n *resourceTranslation) Encode(ctx context.Context, pl *payload) (err error) { + + // @todo move out of the encoding logic + if n == nil { + return + } + localeRes, err := n.makeResourceTranslation(pl) if err != nil { return err @@ -118,6 +125,15 @@ func (n *resourceTranslation) makeResourceTranslation(pl *payload) (string, erro _ = localeRes switch n.refLocaleRes.ResourceType { + case automationTypes.WorkflowResourceType: + p1 := resource.FindAutomationWorkflow(pl.state.ParentResources, n.refLocaleRes.Identifiers) + if p1 == nil { + return "", resource.AutomationWorkflowErrUnresolved(n.refLocaleRes.Identifiers) + } + p1ID = p1.ID + + return automationTypes.WorkflowResourceTranslation(p1ID), nil + case composeTypes.NamespaceResourceType: p1 := resource.FindComposeNamespace(pl.state.ParentResources, n.refLocaleRes.Identifiers) if p1 == nil { @@ -126,6 +142,7 @@ func (n *resourceTranslation) makeResourceTranslation(pl *payload) (string, erro p1ID = p1.ID return composeTypes.NamespaceResourceTranslation(p1ID), nil + case composeTypes.ModuleResourceType: p0 := resource.FindComposeNamespace(pl.state.ParentResources, n.refPathRes[0].Identifiers) if p0 == nil { @@ -156,6 +173,21 @@ func (n *resourceTranslation) makeResourceTranslation(pl *payload) (string, erro return composeTypes.PageResourceTranslation(p0ID, p1ID), nil + case composeTypes.ChartResourceType: + p0 := resource.FindComposeNamespace(pl.state.ParentResources, n.refPathRes[0].Identifiers) + if p0 == nil { + return "", resource.ComposeNamespaceErrUnresolved(n.refPathRes[0].Identifiers) + } + p0ID = p0.ID + + p1 := resource.FindComposeChart(pl.state.ParentResources, n.refLocaleRes.Identifiers) + if p1 == nil { + return "", resource.ComposeChartErrUnresolved(n.refLocaleRes.Identifiers) + } + p1ID = p1.ID + + return composeTypes.ChartResourceTranslation(p0ID, p1ID), nil + case composeTypes.ModuleFieldResourceType: p0 := resource.FindComposeNamespace(pl.state.ParentResources, n.refPathRes[0].Identifiers) if p0 == nil { @@ -181,7 +213,7 @@ func (n *resourceTranslation) makeResourceTranslation(pl *payload) (string, erro default: // @todo if we wish to support res. trans. for external stuff, this needs to pass through. // this also requires some tweaks in the path ID thing. - return "", fmt.Errorf("unsupported resource type '%s' for RBAC store encode", n.refLocaleRes.ResourceType) + return "", fmt.Errorf("unsupported resource type '%s' for resource translation store encode", n.refLocaleRes.ResourceType) } } diff --git a/pkg/envoy/store/system.go b/pkg/envoy/store/system.go index 21bd9bf7f..4f6c93448 100644 --- a/pkg/envoy/store/system.go +++ b/pkg/envoy/store/system.go @@ -483,6 +483,50 @@ func (df *DecodeFilter) systemFromResource(rr ...string) *DecodeFilter { return df } +func (df *DecodeFilter) systemFromRef(rr ...*resource.Ref) *DecodeFilter { + for _, r := range rr { + if strings.Index(r.ResourceType, "system") < 0 { + continue + } + + switch r.ResourceType { + case types.RoleResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.Roles(&types.RoleFilter{ + Query: i, + }) + } + case types.UserResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.Users(&types.UserFilter{ + Query: i, + AllKinds: true, + }) + } + case types.TemplateResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.Templates(&types.TemplateFilter{ + Handle: i, + }) + templateID, err := cast.ToUint64E(i) + if err == nil && templateID > 0 { + df = df.Templates(&types.TemplateFilter{ + TemplateID: []uint64{templateID}, + }) + } + } + case types.ApplicationResourceType: + for _, i := range r.Identifiers.StringSlice() { + df = df.Applications(&types.ApplicationFilter{ + Query: i, + }) + } + } + } + + return df +} + // Roles adds a new RoleFilter func (df *DecodeFilter) Roles(f *types.RoleFilter) *DecodeFilter { if df.roles == nil { diff --git a/pkg/envoy/yaml/encoder.go b/pkg/envoy/yaml/encoder.go index 449112013..412c1291f 100644 --- a/pkg/envoy/yaml/encoder.go +++ b/pkg/envoy/yaml/encoder.go @@ -108,6 +108,11 @@ func (ye *yamlEncoder) Prepare(ctx context.Context, ee ...*envoy.ResourceState) } for _, e := range ee { + // Skip placeholders + if e.Res.Placeholder() { + continue + } + switch res := e.Res.(type) { // Compose resources case *resource.ComposeNamespace: @@ -176,6 +181,11 @@ func (ye *yamlEncoder) Encode(ctx context.Context, p envoy.Provider) error { break } + // Skip placeholders + if e.Res.Placeholder() { + continue + } + state := ye.resState[e.Res] if state == nil { err = ErrResourceStateUndefined diff --git a/pkg/envoy/yaml/resource_translation.go b/pkg/envoy/yaml/resource_translation.go index 37d3b02a6..8e835a4a4 100644 --- a/pkg/envoy/yaml/resource_translation.go +++ b/pkg/envoy/yaml/resource_translation.go @@ -43,6 +43,10 @@ func (ll resourceTranslationSet) groupByResourceTranslation() (out resourceTrans } func resourceTranslationFromResource(r *resource.ResourceTranslation, cfg *EncoderConfig) *resourceTranslation { + if len(r.Res) == 0 { + return nil + } + return &resourceTranslation{ locales: r.Res, diff --git a/pkg/envoy/yaml/resource_translation_marshal.go b/pkg/envoy/yaml/resource_translation_marshal.go index 3cbbb7f40..4d7b502f8 100644 --- a/pkg/envoy/yaml/resource_translation_marshal.go +++ b/pkg/envoy/yaml/resource_translation_marshal.go @@ -62,6 +62,11 @@ func (r *resourceTranslation) Encode(ctx context.Context, doc *Document, state * // under the related namespace. // For now all rules will be nested under a root node for simplicity sake. + // @todo move out of the encoding logic + if r == nil { + return + } + refResource, err := r.makeResourceTranslationResource(state) if err != nil { return err diff --git a/store/tests/resource_translations_test.go b/store/tests/resource_translations_test.go new file mode 100644 index 000000000..f35938fbe --- /dev/null +++ b/store/tests/resource_translations_test.go @@ -0,0 +1,11 @@ +package tests + +import ( + "testing" + + "github.com/cortezaproject/corteza-server/store" +) + +func testResourceTranslation(t *testing.T, s store.Reports) { + t.Skip("@todo") +} diff --git a/tests/envoy/main.go b/tests/envoy/main.go index da22cf606..b8271b88b 100644 --- a/tests/envoy/main.go +++ b/tests/envoy/main.go @@ -63,12 +63,31 @@ func decodeDirectory(ctx context.Context, suite string) ([]resource.Interface, e func encode(ctx context.Context, s store.Storer, nn []resource.Interface) error { se := es.NewStoreEncoder(s, nil) - bld := envoy.NewBuilder(se) - g, err := bld.Build(ctx, nn...) - if err != nil { + g, err := envoy.NewSafeBuilder(se).Build(ctx, nn...) + if err != nil && err != envoy.BuilderErrUnresolvedReferences { return err } + if err == envoy.BuilderErrUnresolvedReferences { + md := g.MissingDeps().Unique() + df := es.NewDecodeFilter().FromRef(md...) + + sd := es.Decoder() + mm, err := sd.Decode(ctx, s, df) + if err != nil { + return err + } + + for _, m := range mm { + m.MarkPlaceholder() + } + + g, err = envoy.NewBuilder(se).Build(ctx, append(nn, mm...)...) + if err != nil { + return err + } + } + return envoy.Encode(ctx, g, se) }