From 8e679bf97ce18f67664b375453d3db3f31b78895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Fri, 25 Feb 2022 16:47:33 +0100 Subject: [PATCH] Improve namespace export * Add RBAC rules * add resource translations * slight refactor --- compose/rest/namespace.go | 299 +++++++++++++++++++++++++---- compose/service/namespace.go | 99 ---------- pkg/envoy/resource/compose_page.go | 13 ++ pkg/rbac/rule.go | 11 ++ tests/compose/main_test.go | 5 + 5 files changed, 295 insertions(+), 132 deletions(-) diff --git a/compose/rest/namespace.go b/compose/rest/namespace.go index 59f87342e..b1dd332c4 100644 --- a/compose/rest/namespace.go +++ b/compose/rest/namespace.go @@ -1,12 +1,17 @@ package rest import ( + "archive/zip" + "bytes" "context" "fmt" "io" + "io/ioutil" "net/http" + "strconv" "time" + automationTypes "github.com/cortezaproject/corteza-server/automation/types" "github.com/cortezaproject/corteza-server/compose/rest/request" "github.com/cortezaproject/corteza-server/compose/service" "github.com/cortezaproject/corteza-server/compose/service/event" @@ -18,6 +23,10 @@ import ( envoyStore "github.com/cortezaproject/corteza-server/pkg/envoy/store" "github.com/cortezaproject/corteza-server/pkg/envoy/yaml" "github.com/cortezaproject/corteza-server/pkg/filter" + "github.com/cortezaproject/corteza-server/pkg/locale" + "github.com/cortezaproject/corteza-server/pkg/rbac" + systemService "github.com/cortezaproject/corteza-server/system/service" + systemTypes "github.com/cortezaproject/corteza-server/system/types" ) type ( @@ -38,10 +47,22 @@ type ( Set []*namespacePayload `json:"set"` } + pageFinder interface { + Find(ctx context.Context, filter types.PageFilter) (set types.PageSet, f types.PageFilter, err error) + } + + chartFinder interface { + Find(ctx context.Context, filter types.ChartFilter) (set types.ChartSet, f types.ChartFilter, err error) + } + Namespace struct { namespace service.NamespaceService + module service.ModuleService + page pageFinder + chart chartFinder locale service.ResourceTranslationsManagerService attachment service.AttachmentService + role systemService.RoleService ac namespaceAccessController } @@ -61,7 +82,11 @@ type ( func (Namespace) New() *Namespace { return &Namespace{ namespace: service.DefaultNamespace, + module: service.DefaultModule, + page: service.DefaultPage, + chart: service.DefaultChart, locale: service.DefaultResourceTranslation, + role: systemService.DefaultRole, attachment: service.DefaultAttachment, ac: service.DefaultAccessControl, } @@ -213,49 +238,79 @@ func (ctrl Namespace) Clone(ctx context.Context, r *request.NamespaceClone) (int return ctrl.makePayload(ctx, ns, err) } -func (ctrl Namespace) Export(ctx context.Context, r *request.NamespaceExport) (interface{}, error) { +func (ctrl Namespace) Export(ctx context.Context, r *request.NamespaceExport) (out interface{}, err error) { var ( - // @todo support multiple archive types - ext = "zip" - file = fmt.Sprintf("%s.%s", r.Filename, ext) + resources resource.InterfaceSet ) - // prepare filters - df := envoyStore.NewDecodeFilter() - - // - compose resources - df = df.ComposeNamespace(&types.NamespaceFilter{ - NamespaceID: []uint64{r.NamespaceID}, - }). - ComposeModule(&types.ModuleFilter{}). - ComposePage(&types.PageFilter{}). - ComposeChart(&types.ChartFilter{}) - - // - workflow - // @todo how do we want to handle these ones? - // do we handle these ones? - - decoder := func() (resource.InterfaceSet, error) { - // get from store - sd := envoyStore.Decoder() - return sd.Decode(ctx, service.DefaultStore, df) + // Prepare resources + resources, nsII, err := ctrl.exportCompose(ctx, r.NamespaceID) + if err != nil { + return } - encoder := func(nn resource.InterfaceSet) (envoy.Streamer, error) { - // prepare for encoding - ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{}) - bld := envoy.NewBuilder(ye) - g, err := bld.Build(ctx, nn...) + // Tweak exported resources + resources = ctrl.tweakExport(ctx, resources, nsII) + + // RBAC + auxRBAC, err := ctrl.exportRBAC(ctx, resources) + if err != nil { + return + } + + // Translations + auxResTrans, err := ctrl.exportResourceTranslations(ctx, resources) + if err != nil { + return + } + resources = append(resources, auxRBAC...) + resources = append(resources, auxResTrans...) + + // Encode + ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{}) + bld := envoy.NewBuilder(ye) + g, err := bld.Build(ctx, resources...) + if err != nil { + return nil, err + } + + err = envoy.Encode(ctx, g, ye) + if err != nil { + return + } + + // Archive encoded resources + buf := bytes.NewBuffer(nil) + w := zip.NewWriter(buf) + + var ( + f io.Writer + bb []byte + ) + for _, s := range ye.Stream() { + // @todo generalize when needed + f, err = w.Create(fmt.Sprintf("%s.yaml", s.Resource)) if err != nil { - return nil, err + return } - err = envoy.Encode(ctx, g, ye) - return ye, err + bb, err = ioutil.ReadAll(s.Source) + if err != nil { + return + } + + _, err = f.Write(bb) + if err != nil { + return + } } - rs, err := ctrl.namespace.Export(ctx, r.NamespaceID, ext, decoder, encoder) - return ctrl.serveExport(ctx, file, rs, err) + err = w.Close() + if err != nil { + return + } + + return ctrl.serveExport(ctx, fmt.Sprintf("%s.zip", r.Filename), bytes.NewReader(buf.Bytes()), nil) } func (ctrl Namespace) ImportInit(ctx context.Context, r *request.NamespaceImportInit) (interface{}, error) { @@ -354,3 +409,181 @@ func (ctrl Namespace) makeFilterPayload(ctx context.Context, nn types.NamespaceS return nsp, nil } + +func (ctrl Namespace) exportCompose(ctx context.Context, namespaceID uint64) (resources resource.InterfaceSet, nsII resource.Identifiers, err error) { + // - namespace + n, err := ctrl.namespace.FindByID(ctx, namespaceID) + if err != nil { + return + } + nsRes := resource.NewComposeNamespace(n) + nsII = nsRes.Identifiers() + resources = append(resources, nsRes) + // - modules + mm, _, err := ctrl.module.Find(ctx, types.ModuleFilter{NamespaceID: n.ID}) + if err != nil { + return + } + for _, m := range mm { + resources = append(resources, resource.NewComposeModule(m, n.Slug)) + } + // - pages + pp, _, err := ctrl.page.Find(ctx, types.PageFilter{NamespaceID: n.ID}) + if err != nil { + return + } + for _, p := range pp { + p, modRef, parentRef := resource.UnpackComposePage(p) + resources = append(resources, resource.NewComposePage(p, n.Slug, modRef, parentRef)) + } + // - charts + cc, _, err := ctrl.chart.Find(ctx, types.ChartFilter{NamespaceID: n.ID}) + if err != nil { + return + } + for _, c := range cc { + refMods := make([]string, 0, 2) + for _, r := range c.Config.Reports { + refMods = append(refMods, strconv.FormatUint(r.ModuleID, 10)) + } + resources = append(resources, resource.NewComposeChart(c, n.Slug, refMods)) + } + + return +} + +func (ctrl Namespace) exportRBAC(ctx context.Context, base resource.InterfaceSet) (resources resource.InterfaceSet, err error) { + // get all roles; we'll filter as needed later + roleIndex := make(map[uint64]*systemTypes.Role) + rr, _, err := ctrl.role.Find(ctx, systemTypes.RoleFilter{}) + if err != nil { + return + } + for _, r := range rr { + roleIndex[r.ID] = r + } + + // get all rbac rules + rules := rbac.Global().Rules() + + // Match up + dupRoleIndex := make(map[uint64]bool) + dupRuleIndex := make(map[string]bool) + for _, res := range base { + // Skip if we can't handle + res, ok := res.(resource.RBACInterface) + if !ok { + continue + } + + rbacRes, _, _ := res.RBACParts() + filteredRules := rules.FilterResource(rbacRes) + for _, rule := range filteredRules { + k := fmt.Sprintf("%s, %s, %d; %d", rule.Resource, rule.Operation, rule.Access, rule.RoleID) + if dupRuleIndex[k] { + continue + } + + _, ref, pp, err := resource.ParseRule(rule.Resource) + if err != nil { + return nil, err + } + + role, ok := roleIndex[rule.RoleID] + if !ok { + continue + } + if !dupRoleIndex[role.ID] { + // Add a placeholder role so Envoy can resolve dependencies + r := resource.NewRole(role) + r.MarkPlaceholder() + resources = append(resources, r) + dupRoleIndex[role.ID] = true + } + + roleRef := role.Handle + if roleRef == "" { + roleRef = strconv.FormatUint(role.ID, 10) + } + + dupRuleIndex[k] = true + resources = append(resources, resource.NewRbacRule( + rule, + roleRef, + ref, + rule.Resource, + pp..., + )) + } + } + + return +} + +func (ctrl Namespace) exportResourceTranslations(ctx context.Context, base resource.InterfaceSet) (resources resource.InterfaceSet, err error) { + lsvc := locale.Global() + tags := lsvc.Tags() + + for _, res := range base { + // Skip if we can't handle + res, ok := res.(resource.LocaleInterface) + if !ok { + continue + } + + // Collect available resource translations + for _, t := range tags { + transRes, _, _ := res.ResourceTranslationParts() + + translations := make(systemTypes.ResourceTranslationSet, 0, 4) + for _, trans := range locale.Global().ResourceTranslations(t, transRes) { + translations = append(translations, systemTypes.FromLocale(locale.ResourceTranslationSet{trans})...) + } + + for _, trans := range locale.Global().ResourceTranslations(t, transRes) { + _, ref, pp, err := resource.ParseResourceTranslation(trans.Resource) + if err != nil { + return nil, err + } + + resources = append(resources, resource.NewResourceTranslation( + translations, + ref.Identifiers.First(), + ref, + pp..., + )) + } + } + } + + return +} + +func (ctrl Namespace) tweakExport(ctx context.Context, resources resource.InterfaceSet, nsII resource.Identifiers) resource.InterfaceSet { + oldNsRef := resource.MakeRef(types.NamespaceResourceType, nsII) + prune := resource.RefSet{resource.MakeWildRef(automationTypes.WorkflowResourceType)} + ns := resource.FindComposeNamespace(resources, nsII) + + // - remove logo and icon references as attachments are not exported by default + // @todo code in attachment exporting, most likely when we do attachment handling rework + ns.Meta.Icon = "" + ns.Meta.IconID = 0 + ns.Meta.Logo = "" + ns.Meta.LogoID = 0 + ns.Meta.LogoEnabled = false + + // - prune resources we won't preserve + resources.SearchForReferences(oldNsRef).Walk(func(r resource.Interface) error { + pp, ok := r.(resource.PrunableInterface) + if !ok { + return nil + } + + for _, p := range prune { + pp.Prune(p) + } + return nil + }) + + return resources +} diff --git a/compose/service/namespace.go b/compose/service/namespace.go index 2c6c447ba..d3b6fbe09 100644 --- a/compose/service/namespace.go +++ b/compose/service/namespace.go @@ -2,11 +2,7 @@ package service import ( "archive/zip" - "bytes" "context" - "fmt" - "io" - "io/ioutil" "mime/multipart" "reflect" "strconv" @@ -18,7 +14,6 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/actionlog" "github.com/cortezaproject/corteza-server/pkg/auth" - "github.com/cortezaproject/corteza-server/pkg/envoy" "github.com/cortezaproject/corteza-server/pkg/envoy/resource" "github.com/cortezaproject/corteza-server/pkg/envoy/yaml" "github.com/cortezaproject/corteza-server/pkg/errors" @@ -76,7 +71,6 @@ type ( Create(ctx context.Context, namespace *types.Namespace) (*types.Namespace, error) Update(ctx context.Context, namespace *types.Namespace) (*types.Namespace, error) Clone(ctx context.Context, namespaceID uint64, dup *types.Namespace, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) - Export(ctx context.Context, namespaceID uint64, archive string, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) (envoy.Streamer, error)) (r io.ReadSeeker, err error) ImportInit(ctx context.Context, f multipart.File, size int64) (namespaceImportSession, error) ImportRun(ctx context.Context, sessionID uint64, dup *types.Namespace, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) DeleteByID(ctx context.Context, namespaceID uint64) error @@ -358,99 +352,6 @@ func (svc namespace) Clone(ctx context.Context, namespaceID uint64, dup *types.N return dup, svc.recordAction(ctx, aProps, NamespaceActionClone, err) } -func (svc namespace) Export(ctx context.Context, namespaceID uint64, archive string, decoder func() (resource.InterfaceSet, error), encoder func(resource.InterfaceSet) (envoy.Streamer, error)) (r io.ReadSeeker, err error) { - var ( - aProps = &namespaceActionProps{archiveFormat: archive} - ) - - // make archive - buf := bytes.NewBuffer(nil) - w := zip.NewWriter(buf) - - err = func() error { - if archive != "zip" { - return NamespaceErrUnsupportedExportFormat() - } - - // initial validation - // - target namespace - targetNs, err := store.LookupComposeNamespaceByID(ctx, svc.store, namespaceID) - if errors.IsNotFound(err) { - return NamespaceErrNotFound() - } else if err != nil { - return err - } - aProps.setNamespace(targetNs) - - // - ac - if err = svc.canExport(ctx, targetNs); err != nil { - return err - } - - // get namespace resources - nn, err := decoder() - if err != nil { - return err - } - - // some meta bits - sNsID := strconv.FormatUint(namespaceID, 10) - oldNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(sNsID)) - prune := resource.RefSet{resource.MakeWildRef(automationTypes.WorkflowResourceType)} - ns := resource.FindComposeNamespace(nn, resource.MakeIdentifiers(sNsID)) - - // - remove logo and icon references as attachments are not exported by default - // @todo code in attachment exporting, most likely when we do attachment handling rework - ns.Meta.Icon = "" - ns.Meta.IconID = 0 - ns.Meta.Logo = "" - ns.Meta.LogoID = 0 - ns.Meta.LogoEnabled = false - - // - prune resources we won't preserve - nn.SearchForReferences(oldNsRef).Walk(func(r resource.Interface) error { - pp, ok := r.(resource.PrunableInterface) - if !ok { - return nil - } - - for _, p := range prune { - pp.Prune(p) - } - return nil - }) - - // encode - ss, err := encoder(nn) - if err != nil { - return err - } - - // create archive - for _, s := range ss.Stream() { - // @todo generalize when needed - f, err := w.Create(fmt.Sprintf("%s.yaml", s.Resource)) - if err != nil { - return err - } - - bb, err := ioutil.ReadAll(s.Source) - if err != nil { - return err - } - - _, err = f.Write(bb) - if err != nil { - return err - } - } - - return w.Close() - }() - - return bytes.NewReader(buf.Bytes()), svc.recordAction(ctx, aProps, NamespaceActionExport, err) -} - func (svc namespace) ImportInit(ctx context.Context, f multipart.File, size int64) (namespaceImportSession, error) { var ( aProps = &namespaceActionProps{} diff --git a/pkg/envoy/resource/compose_page.go b/pkg/envoy/resource/compose_page.go index cb88d0059..e66abb26e 100644 --- a/pkg/envoy/resource/compose_page.go +++ b/pkg/envoy/resource/compose_page.go @@ -129,6 +129,19 @@ func NewComposePage(pg *types.Page, nsRef, modRef, parentRef string) *ComposePag return r } +func UnpackComposePage(p *types.Page) (*types.Page, string, string) { + modRef := "" + parentRef := "" + if p.ModuleID != 0 { + modRef = strconv.FormatUint(p.ModuleID, 10) + } + if p.SelfID != 0 { + parentRef = strconv.FormatUint(p.SelfID, 10) + } + + return p, modRef, parentRef +} + func (r *ComposePage) Resource() interface{} { return r.Res } diff --git a/pkg/rbac/rule.go b/pkg/rbac/rule.go index 134b70cfa..27ad04ca0 100644 --- a/pkg/rbac/rule.go +++ b/pkg/rbac/rule.go @@ -60,6 +60,17 @@ func (set RuleSet) FilterAccess(a Access) (out RuleSet) { return out } +func (set RuleSet) FilterResource(res string) (out RuleSet) { + for _, r := range set { + if !matchResource(r.Resource, res) { + continue + } + out = append(out, r) + } + + return +} + // AllowRule helper func to create allow rule func AllowRule(id uint64, r, o string) *Rule { return &Rule{id, r, o, Allow, false} diff --git a/tests/compose/main_test.go b/tests/compose/main_test.go index 03d03d7e1..b23ff6b0e 100644 --- a/tests/compose/main_test.go +++ b/tests/compose/main_test.go @@ -267,10 +267,15 @@ func grantImportExport(h helper) { helpers.AllowMe(h, types.ComponentRbacResource(), "module.create") helpers.AllowMe(h, types.ComponentRbacResource(), "page.create") helpers.AllowMe(h, types.ComponentRbacResource(), "chart.create") + helpers.AllowMe(h, types.NamespaceRbacResource(0), "modules.search") + helpers.AllowMe(h, types.NamespaceRbacResource(0), "charts.search") + helpers.AllowMe(h, types.NamespaceRbacResource(0), "pages.search") helpers.AllowMe(h, types.NamespaceRbacResource(0), "read") helpers.AllowMe(h, types.ModuleRbacResource(0, 0), "read") helpers.AllowMe(h, types.PageRbacResource(0, 0), "read") helpers.AllowMe(h, types.ChartRbacResource(0, 0), "read") + helpers.AllowMe(h, sysTypes.ComponentRbacResource(), "roles.search") + helpers.AllowMe(h, sysTypes.RoleRbacResource(0), "read") } func namespaceExportSafe(t *testing.T, h helper, namespaceID uint64) []byte {