From 0853b92393215e77c1b5bb07da0cbe7d6ac288c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Thu, 10 Jun 2021 14:31:00 +0200 Subject: [PATCH] Allow most envy/store encoders to ignore existing resources --- pkg/envoy/store/application_marshal.go | 5 ++ .../store/automation_workflow_marshal.go | 12 ++++ pkg/envoy/store/compose_chart_marshal.go | 25 +++++-- pkg/envoy/store/compose_module_marshal.go | 15 ++-- pkg/envoy/store/compose_namespace_marshal.go | 12 ++-- pkg/envoy/store/compose_page_marshal.go | 68 ++++++++++++++----- pkg/envoy/store/compose_record_marshal.go | 5 ++ pkg/envoy/store/encoder.go | 3 + pkg/envoy/store/role_marshal.go | 5 ++ pkg/envoy/store/system.go | 20 +++++- pkg/envoy/store/template_marshal.go | 5 ++ pkg/envoy/store/user_marshal.go | 5 ++ pkg/envoy/store/util.go | 3 + 13 files changed, 149 insertions(+), 34 deletions(-) diff --git a/pkg/envoy/store/application_marshal.go b/pkg/envoy/store/application_marshal.go index 1f5e53a2d..d798da6df 100644 --- a/pkg/envoy/store/application_marshal.go +++ b/pkg/envoy/store/application_marshal.go @@ -16,6 +16,11 @@ func NewApplicationFromResource(res *resource.Application, cfg *EncoderConfig) r } func (n *application) Prepare(ctx context.Context, pl *payload) (err error) { + if n.cfg.IgnoreStore { + n.res.Res.ID = 0 + return nil + } + // Get the existing app n.app, err = findApplicationStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) if err != nil { diff --git a/pkg/envoy/store/automation_workflow_marshal.go b/pkg/envoy/store/automation_workflow_marshal.go index 8026537f9..52895da62 100644 --- a/pkg/envoy/store/automation_workflow_marshal.go +++ b/pkg/envoy/store/automation_workflow_marshal.go @@ -27,6 +27,11 @@ func (n *automationWorkflow) Prepare(ctx context.Context, pl *payload) (err erro } func (n *automationWorkflow) prepareWorkflows(ctx context.Context, pl *payload) (err error) { + if n.cfg.IgnoreStore { + n.res.Res.ID = 0 + return nil + } + // Try to get the original workflow n.wf, err = findAutomationWorkflowStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) if err != nil { @@ -44,6 +49,13 @@ func (n *automationWorkflow) prepareTriggers(ctx context.Context, pl *payload) ( return nil } + if n.cfg.IgnoreStore { + for _, t := range n.tt { + t.ID = 0 + } + return nil + } + // Try to find any related triggers for this workflow tt, _, err := store.SearchAutomationTriggers(ctx, pl.s, types.TriggerFilter{ WorkflowID: []uint64{n.wf.ID}, diff --git a/pkg/envoy/store/compose_chart_marshal.go b/pkg/envoy/store/compose_chart_marshal.go index a73d481e2..a415a1d5f 100644 --- a/pkg/envoy/store/compose_chart_marshal.go +++ b/pkg/envoy/store/compose_chart_marshal.go @@ -18,9 +18,14 @@ func newComposeChartFromResource(res *resource.ComposeChart, cfg *EncoderConfig) func (n *composeChart) Prepare(ctx context.Context, pl *payload) (err error) { // Get related namespace - n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) + if err != nil { + return err + } + } else { + n.res.Res.ID = 0 + n.relNS = resource.FindComposeNamespace(pl.state.ParentResources, n.res.RefNs.Identifiers) } if n.relNS == nil { return resource.ComposeNamespaceErrUnresolved(n.res.RefNs.Identifiers) @@ -29,15 +34,23 @@ func (n *composeChart) Prepare(ctx context.Context, pl *payload) (err error) { // Get related modules n.relMods = make(types.ModuleSet, len(n.res.RefMods)) for i, rMod := range n.res.RefMods { - n.relMods[i], err = findComposeModule(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, rMod.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + n.relMods[i], err = findComposeModule(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, rMod.Identifiers) + if err != nil { + return err + } + } else { + n.relMods[i] = resource.FindComposeModule(pl.state.ParentResources, rMod.Identifiers) } if n.relMods[i] == nil { return resource.ComposeModuleErrUnresolved(rMod.Identifiers) } } + if n.cfg.IgnoreStore { + return nil + } + // Try to get the original chart n.chr, err = findComposeChartStore(ctx, pl.s, n.relNS.ID, makeGenericFilter(n.res.Identifiers())) if err != nil { diff --git a/pkg/envoy/store/compose_module_marshal.go b/pkg/envoy/store/compose_module_marshal.go index 45284fd2a..69ebaf0d5 100644 --- a/pkg/envoy/store/compose_module_marshal.go +++ b/pkg/envoy/store/compose_module_marshal.go @@ -21,9 +21,14 @@ func NewComposeModuleFromResource(res *resource.ComposeModule, cfg *EncoderConfi func (n *composeModule) Prepare(ctx context.Context, pl *payload) (err error) { // Get related namespace - n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) + if err != nil { + return err + } + } else { + n.res.Res.ID = 0 + n.relNS = resource.FindComposeNamespace(pl.state.ParentResources, n.res.RefNs.Identifiers) } if n.relNS == nil { return resource.ComposeNamespaceErrUnresolved(n.res.RefNs.Identifiers) @@ -34,7 +39,7 @@ func (n *composeModule) Prepare(ctx context.Context, pl *payload) (err error) { // Get related record field modules for _, refMod := range n.res.RefMods { var mod *types.Module - if n.relNS.ID > 0 { + if !n.cfg.IgnoreStore && n.relNS.ID > 0 { mod, err = findComposeModuleStore(ctx, pl.s, n.relNS.ID, makeGenericFilter(refMod.Identifiers)) if err != nil { return err @@ -53,7 +58,7 @@ func (n *composeModule) Prepare(ctx context.Context, pl *payload) (err error) { } // Can't do anything else, since the NS doesn't yet exist - if n.relNS.ID <= 0 { + if n.cfg.IgnoreStore || n.relNS.ID <= 0 { return nil } diff --git a/pkg/envoy/store/compose_namespace_marshal.go b/pkg/envoy/store/compose_namespace_marshal.go index ae72838d3..5186ad349 100644 --- a/pkg/envoy/store/compose_namespace_marshal.go +++ b/pkg/envoy/store/compose_namespace_marshal.go @@ -16,10 +16,14 @@ func newComposeNamespaceFromResource(res *resource.ComposeNamespace, cfg *Encode } func (n *composeNamespace) Prepare(ctx context.Context, pl *payload) (err error) { - // Try to get the original namespace - n.ns, err = findComposeNamespaceStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) - if err != nil { - return err + if !n.cfg.IgnoreStore { + // Try to get the original namespace + n.ns, err = findComposeNamespaceStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) + if err != nil { + return err + } + } else { + n.res.Res.ID = 0 } if n.ns != nil { diff --git a/pkg/envoy/store/compose_page_marshal.go b/pkg/envoy/store/compose_page_marshal.go index 0002c0447..b9ed1ea54 100644 --- a/pkg/envoy/store/compose_page_marshal.go +++ b/pkg/envoy/store/compose_page_marshal.go @@ -24,9 +24,14 @@ func newComposePageFromResource(res *resource.ComposePage, cfg *EncoderConfig) r func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { // Get related namespace - n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) + if err != nil { + return err + } + } else { + n.res.Res.ID = 0 + n.relNS = resource.FindComposeNamespace(pl.state.ParentResources, n.res.RefNs.Identifiers) } if n.relNS == nil { return resource.ComposeNamespaceErrUnresolved(n.res.RefNs.Identifiers) @@ -35,9 +40,13 @@ func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { // Get related module // If this isn't a record page, there is no related module if n.res.RefMod != nil { - n.relMod, err = findComposeModule(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, n.res.RefMod.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + n.relMod, err = findComposeModule(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, n.res.RefMod.Identifiers) + if err != nil { + return err + } + } else { + n.relMod = resource.FindComposeModule(pl.state.ParentResources, n.res.RefMod.Identifiers) } if n.relMod == nil { return resource.ComposeModuleErrUnresolved(n.res.RefMod.Identifiers) @@ -46,9 +55,13 @@ func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { // Get parent page if n.res.RefParent != nil { - n.relParent, err = findComposePage(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, n.res.RefParent.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + n.relParent, err = findComposePage(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, n.res.RefParent.Identifiers) + if err != nil { + return err + } + } else { + n.relParent = resource.FindComposePage(pl.state.ParentResources, n.res.RefParent.Identifiers) } if n.relParent == nil { return resource.ComposePageErrUnresolved(n.res.RefParent.Identifiers) @@ -56,10 +69,15 @@ func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { } // Get related workflows + var wf *atypes.Workflow for _, wfr := range n.res.WfRefs { - wf, err := findAutomationWorkflow(ctx, pl.s, pl.state.ParentResources, wfr.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + wf, err = findAutomationWorkflow(ctx, pl.s, pl.state.ParentResources, wfr.Identifiers) + if err != nil { + return err + } + } else { + wf = resource.FindAutomationWorkflow(pl.state.ParentResources, wfr.Identifiers) } if wf == nil { return resource.AutomationWorkflowErrUnresolved(wfr.Identifiers) @@ -70,10 +88,15 @@ func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { } // Get other related modules + var mod *types.Module for _, mr := range n.res.ModRefs { - mod, err := findComposeModule(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, mr.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + mod, err = findComposeModule(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, mr.Identifiers) + if err != nil { + return err + } + } else { + mod = resource.FindComposeModule(pl.state.ParentResources, mr.Identifiers) } if mod == nil { return resource.ComposeModuleErrUnresolved(mr.Identifiers) @@ -84,10 +107,15 @@ func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { } // Get related charts + var chr *types.Chart for _, refChart := range n.res.RefCharts { - chr, err := findComposeChart(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, refChart.Identifiers) - if err != nil { - return err + if !n.cfg.IgnoreStore { + chr, err = findComposeChart(ctx, pl.s, n.relNS.ID, pl.state.ParentResources, refChart.Identifiers) + if err != nil { + return err + } + } else { + chr = resource.FindComposeChart(pl.state.ParentResources, refChart.Identifiers) } if chr == nil { return resource.ComposeChartErrUnresolved(refChart.Identifiers) @@ -97,6 +125,10 @@ func (n *composePage) Prepare(ctx context.Context, pl *payload) (err error) { } } + if n.cfg.IgnoreStore { + return nil + } + // Try to get the original page n.pg, err = findComposePageStore(ctx, pl.s, n.relNS.ID, makeGenericFilter(n.res.Identifiers())) if err != nil { diff --git a/pkg/envoy/store/compose_record_marshal.go b/pkg/envoy/store/compose_record_marshal.go index ab36414d2..caa6a5322 100644 --- a/pkg/envoy/store/compose_record_marshal.go +++ b/pkg/envoy/store/compose_record_marshal.go @@ -33,6 +33,11 @@ func NewComposeRecordFromResource(res *resource.ComposeRecord, cfg *EncoderConfi } func (n *composeRecord) Prepare(ctx context.Context, pl *payload) (err error) { + // @todo add this + if n.cfg.IgnoreStore { + return fmt.Errorf("IgnoreStore not supported for compose records") + } + // Get related namespace n.relNS, err = findComposeNamespace(ctx, pl.s, pl.state.ParentResources, n.res.RefNs.Identifiers) if err != nil { diff --git a/pkg/envoy/store/encoder.go b/pkg/envoy/store/encoder.go index 41cbeabf9..52dcc2838 100644 --- a/pkg/envoy/store/encoder.go +++ b/pkg/envoy/store/encoder.go @@ -40,6 +40,9 @@ type ( // If you return an error, the encoding will terminate. // If you return nil (ignore the error), the encoding will continue. DeferNok func(error) error + + // IgnoreStore prevents encoders from accessing the store for initial resources + IgnoreStore bool } accessControlRBACServicer interface { diff --git a/pkg/envoy/store/role_marshal.go b/pkg/envoy/store/role_marshal.go index 6d8298180..9c06216ad 100644 --- a/pkg/envoy/store/role_marshal.go +++ b/pkg/envoy/store/role_marshal.go @@ -16,6 +16,11 @@ func NewRoleFromResource(res *resource.Role, cfg *EncoderConfig) resourceState { } func (n *role) Prepare(ctx context.Context, pl *payload) (err error) { + if n.cfg.IgnoreStore { + n.res.Res.ID = 0 + return nil + } + n.rl, err = findRoleStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) if err != nil { return err diff --git a/pkg/envoy/store/system.go b/pkg/envoy/store/system.go index b2de8e80d..bc1a3b0ea 100644 --- a/pkg/envoy/store/system.go +++ b/pkg/envoy/store/system.go @@ -21,6 +21,7 @@ type ( rbac.RuleFilter // This will help us determine what rules for what resources we are able to export resourceID map[uint64]bool + strict bool } systemDecoder struct { @@ -288,7 +289,11 @@ func (d *systemDecoder) decodeRbac(ctx context.Context, s store.Storer, ff []*rb for _, n := range nn { // If not wildcard or is a system rule; check if resource is allowed if n.Resource.HasWildcard() || !n.Resource.IsAppendable() { - mm = append(mm, newRbacRule(n)) + // strict mode prevents non-resource specific roles from being exported. + // mainly used for NS duplication + if !f.strict { + mm = append(mm, newRbacRule(n)) + } } else { id, err := n.Resource.GetID() if err != nil { @@ -419,6 +424,19 @@ func (df *DecodeFilter) Rbac(f *rbac.RuleFilter) *DecodeFilter { return df } +func (df *DecodeFilter) RbacStrict(f *rbac.RuleFilter) *DecodeFilter { + if df.rbac == nil { + df.rbac = make([]*rbacFilter, 0, 1) + } else { + // There can only be a single rbac filter + // since it makes no sense to have multiple of + return df + } + + df.rbac = append(df.rbac, &rbacFilter{RuleFilter: *f, strict: true}) + return df +} + // allowRbacResource adds a new resource identifier to supported resource rules func (df *DecodeFilter) allowRbacResource(id ...uint64) { if df.rbac == nil || len(df.rbac) == 0 { diff --git a/pkg/envoy/store/template_marshal.go b/pkg/envoy/store/template_marshal.go index d89e36491..1e98f9592 100644 --- a/pkg/envoy/store/template_marshal.go +++ b/pkg/envoy/store/template_marshal.go @@ -16,6 +16,11 @@ func NewTemplateFromResource(res *resource.Template, cfg *EncoderConfig) resourc } func (n *template) Prepare(ctx context.Context, pl *payload) (err error) { + if n.cfg.IgnoreStore { + n.res.Res.ID = 0 + return nil + } + // Try to get the original template n.t, err = findTemplateStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) if err != nil { diff --git a/pkg/envoy/store/user_marshal.go b/pkg/envoy/store/user_marshal.go index 124e39dc9..e30fc7c1f 100644 --- a/pkg/envoy/store/user_marshal.go +++ b/pkg/envoy/store/user_marshal.go @@ -16,6 +16,11 @@ func NewUserFromResource(res *resource.User, cfg *EncoderConfig) resourceState { } func (n *user) Prepare(ctx context.Context, pl *payload) (err error) { + if n.cfg.IgnoreStore { + n.res.Res.ID = 0 + return nil + } + // Try to get the original user n.u, err = findUserStore(ctx, pl.s, makeGenericFilter(n.res.Identifiers())) if err != nil { diff --git a/pkg/envoy/store/util.go b/pkg/envoy/store/util.go index 1552cc42f..75b5f044f 100644 --- a/pkg/envoy/store/util.go +++ b/pkg/envoy/store/util.go @@ -141,6 +141,9 @@ func mergeConfig(ec *EncoderConfig, rs *resource.EnvoyConfig) *EncoderConfig { rr := &EncoderConfig{ OnExisting: rs.OnExisting, SkipIf: rs.SkipIf, + + // IgnoreStore is an encoder thing and should not be controlled from a resource + IgnoreStore: ec.IgnoreStore, } // Default to store config