From 9f7e87f30e2e7b3021a18e18bc6c86015659efb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Wed, 27 Apr 2022 13:15:06 +0200 Subject: [PATCH] Refactor NS import/export for above changes --- compose/rest/namespace.go | 205 ++++++++++++++++++----------------- compose/service/namespace.go | 90 +++++++++------ pkg/locale/load.go | 9 +- pkg/locale/service.go | 2 +- 4 files changed, 166 insertions(+), 140 deletions(-) diff --git a/compose/rest/namespace.go b/compose/rest/namespace.go index d3662d352..840563fff 100644 --- a/compose/rest/namespace.go +++ b/compose/rest/namespace.go @@ -8,7 +8,6 @@ import ( "io" "io/ioutil" "net/http" - "strconv" "time" automationTypes "github.com/cortezaproject/corteza-server/automation/types" @@ -240,39 +239,12 @@ func (ctrl Namespace) Clone(ctx context.Context, r *request.NamespaceClone) (int } func (ctrl Namespace) Export(ctx context.Context, r *request.NamespaceExport) (out interface{}, err error) { - var ( - resources resource.InterfaceSet - ) - - // Prepare resources - resources, nsII, err := ctrl.exportCompose(ctx, r.NamespaceID) + // Get resources + resources, err := ctrl.gatherResources(ctx, r.NamespaceID) if err != nil { return } - // Tweak exported resources - resources = ctrl.tweakExport(ctx, resources, nsII) - - var roleIndex map[uint64]*systemTypes.Role - resources, roleIndex, err = ctrl.preparePlaceholders(ctx, resources) - if err != nil { - return - } - - // RBAC - auxRBAC, err := ctrl.exportRBAC(ctx, roleIndex, resources) - if err != nil { - return - } - - // Translations - auxResTrans, err := ctrl.exportResourceTranslations(ctx, resources) - if err != nil { - return - } - resources = append(resources, auxRBAC...) - resources = append(resources, auxResTrans...) - // Encode ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{}) bld := envoy.NewBuilder(ye) @@ -417,6 +389,44 @@ func (ctrl Namespace) makeFilterPayload(ctx context.Context, nn types.NamespaceS return nsp, nil } +func (ctrl Namespace) gatherResources(ctx context.Context, namespaceID uint64) (resources resource.InterfaceSet, err error) { + var ( + nsII resource.Identifiers + ) + + // Prepare resources + resources, nsII, err = ctrl.exportCompose(ctx, namespaceID) + if err != nil { + return + } + + // Tweak exported resources + resources = ctrl.tweakExport(ctx, resources, nsII) + + // Role placeholders for RBAC + var roleIndex map[uint64]*systemTypes.Role + resources, roleIndex, err = ctrl.preparePlaceholders(ctx, resources) + if err != nil { + return + } + + // RBAC + auxRBAC, err := ctrl.exportRBAC(ctx, roleIndex, resources) + if err != nil { + return + } + + // Translations + auxResTrans, err := ctrl.exportResourceTranslations(ctx, resources) + if err != nil { + return + } + resources = append(resources, auxRBAC...) + resources = append(resources, auxResTrans...) + + return +} + func (ctrl Namespace) exportCompose(ctx context.Context, namespaceID uint64) (resources resource.InterfaceSet, nsII resource.Identifiers, err error) { // - namespace n, err := ctrl.namespace.FindByID(ctx, namespaceID) @@ -432,7 +442,11 @@ func (ctrl Namespace) exportCompose(ctx context.Context, namespaceID uint64) (re return } for _, m := range mm { - resources = append(resources, resource.NewComposeModule(m, n.Slug)) + km := resource.NewComposeModule(m, resource.MakeNamespaceRef(n.ID, n.Slug, n.Name)) + for _, f := range m.Fields { + km.AddField(resource.NewComposeModuleField(f, km.RefNs, km.Ref())) + } + resources = append(resources, km) } // - pages pp, _, err := ctrl.page.Find(ctx, types.PageFilter{NamespaceID: n.ID}) @@ -441,7 +455,12 @@ func (ctrl Namespace) exportCompose(ctx context.Context, namespaceID uint64) (re } for _, p := range pp { p, modRef, parentRef := resource.UnpackComposePage(p) - resources = append(resources, resource.NewComposePage(p, n.Slug, modRef, parentRef)) + resources = append(resources, resource.NewComposePage( + p, + resource.MakeNamespaceRef(n.ID, n.Slug, n.Name), + modRef, + parentRef, + )) } // - charts cc, _, err := ctrl.chart.Find(ctx, types.ChartFilter{NamespaceID: n.ID}) @@ -449,102 +468,90 @@ func (ctrl Namespace) exportCompose(ctx context.Context, namespaceID uint64) (re return } for _, c := range cc { - refMods := make([]string, 0, 2) + refMods := make(resource.RefSet, 0, 2) for _, r := range c.Config.Reports { - refMods = append(refMods, strconv.FormatUint(r.ModuleID, 10)) + refMods = append(refMods, resource.MakeModuleRef(r.ModuleID, "", "")) } - resources = append(resources, resource.NewComposeChart(c, n.Slug, refMods)) + resources = append(resources, resource.NewComposeChart( + c, + resource.MakeNamespaceRef(n.ID, n.Slug, n.Name), + refMods, + )) } return } func (ctrl Namespace) exportRBAC(ctx context.Context, roleIndex map[uint64]*systemTypes.Role, base resource.InterfaceSet) (resources resource.InterfaceSet, err error) { - // get all rbac rules - rules := rbac.Global().Rules() + // Prepare RBAC Rules + rawRules := rbac.Global().Rules() + rules := make([]*resource.RbacRule, 0, len(rawRules)) + for _, rule := range rawRules { + _, ref, pp, err := resource.ParseRule(rule.Resource) + if err != nil { + return nil, err + } - // Match up - dupRuleIndex := make(map[string]bool) - for _, res := range base { - // Skip if we can't handle - res, ok := res.(resource.RBACInterface) + role, ok := roleIndex[rule.RoleID] if !ok { continue } - rbacRes, _, _ := res.RBACParts() - filteredRules := rules.FilterResource(rbacRes) - for _, rule := range filteredRules { - k := fmt.Sprintf("%s, %s, %d; %d", rule.Resource, rule.Operation, rule.Access, rule.RoleID) - if dupRuleIndex[k] { - continue - } + rules = append(rules, resource.NewRbacRule( + rule, + resource.MakeRoleRef(role.ID, role.Handle, role.Name), + ref, + rule.Resource, + pp..., + )) + } - _, ref, pp, err := resource.ParseRule(rule.Resource) - if err != nil { - return nil, err - } - - role, ok := roleIndex[rule.RoleID] - if !ok { - continue - } - - roleRef := role.Handle - if roleRef == "" { - roleRef = strconv.FormatUint(role.ID, 10) - } - - dupRuleIndex[k] = true - resources = append(resources, resource.NewRbacRule( - rule, - roleRef, - ref, - rule.Resource, - pp..., - )) - } + for _, r := range envoy.FilterRequestedRBACRules(base, rules) { + resources = append(resources, r) } return } func (ctrl Namespace) exportResourceTranslations(ctx context.Context, base resource.InterfaceSet) (resources resource.InterfaceSet, err error) { - lsvc := locale.Global() - tags := lsvc.Tags() + var ( + lsvc = locale.Global() + tags = lsvc.Tags() + translations = make([]*resource.ResourceTranslation, 0, 124) - for _, res := range base { - // Skip if we can't handle - res, ok := res.(resource.LocaleInterface) - if !ok { - continue + resKeyTrans map[string]map[string]*locale.ResourceTranslation + ) + + for _, t := range tags { + resKeyTrans, err = lsvc.LoadResourceTranslations(ctx, t) + if err != nil { + return } - // Collect available resource translations - for _, t := range tags { - transRes, _, _ := res.ResourceTranslationParts() - - translations := make(systemTypes.ResourceTranslationSet, 0, 4) - for _, trans := range locale.Global().ResourceTranslations(t, transRes) { - translations = append(translations, systemTypes.FromLocale(locale.ResourceTranslationSet{trans})...) + for transRes, keyTrans := range resKeyTrans { + rawTranslations := make(locale.ResourceTranslationSet, 0, len(resKeyTrans)) + for _, trans := range keyTrans { + rawTranslations = append(rawTranslations, trans) } - for _, trans := range locale.Global().ResourceTranslations(t, transRes) { - _, ref, pp, err := resource.ParseResourceTranslation(trans.Resource) - if err != nil { - return nil, err - } - - resources = append(resources, resource.NewResourceTranslation( - translations, - ref.Identifiers.First(), - ref, - pp..., - )) + _, ref, pp, err := resource.ParseResourceTranslation(transRes) + if err != nil { + return nil, err } + + translations = append(translations, resource.NewResourceTranslation( + systemTypes.FromLocale(rawTranslations), + ref.Identifiers.First(), + ref, + pp..., + )) } } + for _, t := range envoy.FilterRequiredResourceTranslations(base, translations) { + resources = append(resources, t) + } + return } diff --git a/compose/service/namespace.go b/compose/service/namespace.go index bfdd5518a..2eb09dd05 100644 --- a/compose/service/namespace.go +++ b/compose/service/namespace.go @@ -469,6 +469,7 @@ func (svc namespace) ImportRun(ctx context.Context, sessionID uint64, dup *types var ( session namespaceImportSession ok bool + newNS *types.Namespace ) if session, ok = namespaceSessionStore[sessionID]; !ok { return NamespaceErrImportSessionNotFound() @@ -477,45 +478,14 @@ func (svc namespace) ImportRun(ctx context.Context, sessionID uint64, dup *types delete(namespaceSessionStore, sessionID) }() - // Handle renames and references - oldNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(session.Slug, session.Name)) - newNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(dup.Slug, dup.Name)) - - auxNs := resource.FindComposeNamespace(session.Resources, oldNsRef.Identifiers) - auxNs.ID = 0 - auxNs.Name = dup.Name - auxNs.Slug = dup.Slug - dup = auxNs aProps.setNamespace(dup) - // Correct internal references - // - namespace identifiers - session.Resources.SearchForIdentifiers(oldNsRef.ResourceType, oldNsRef.Identifiers).Walk(func(r resource.Interface) error { - r.ReID(newNsRef.Identifiers) - return nil - }) - // - relations - session.Resources.SearchForReferences(oldNsRef).Walk(func(r resource.Interface) error { - r.ReRef(resource.RefSet{oldNsRef}, resource.RefSet{newNsRef}) - return nil - }) - - // run the import - err = encoder(session.Resources) + newNS, err = svc.envoyRun(ctx, session.Resources, &types.Namespace{Slug: session.Slug, Name: session.Name}, dup, encoder) if err != nil { return err } - // Reload RBAC rules (in case import brought in something new) - rbac.Global().Reload(ctx) - - // Reload workflow-triggers (in case import brought in something new) - if err = automationService.DefaultWorkflow.Load(ctx); err != nil { - // should not be a fatal error - err = nil - } - - aProps.setNamespace(dup) + aProps.setNamespace(newNS) return nil }() @@ -771,6 +741,60 @@ func (svc namespace) canImport(ctx context.Context) error { return nil } +func (svc namespace) envoyRun(ctx context.Context, resources resource.InterfaceSet, oldNS, newNS *types.Namespace, encoder func(resource.InterfaceSet) error) (ns *types.Namespace, err error) { + // Handle renames and references + oldNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(oldNS.Slug, oldNS.Name)) + newNsRef := resource.MakeRef(types.NamespaceResourceType, resource.MakeIdentifiers(newNS.Slug, newNS.Name)) + + auxNs := resource.FindComposeNamespace(resources, oldNsRef.Identifiers) + auxNs.ID = 0 + auxNs.Name = newNS.Name + auxNs.Slug = newNS.Slug + newNS = auxNs + ns = newNS + + // Correct internal references + // - namespace identifiers + resources.SearchForIdentifiers(oldNsRef.ResourceType, oldNsRef.Identifiers).Walk(func(r resource.Interface) error { + r.ReID(newNsRef.Identifiers) + return nil + }) + // - relations + resources.SearchForReferences(oldNsRef).Walk(func(r resource.Interface) error { + r.ReRef(resource.RefSet{oldNsRef}, resource.RefSet{newNsRef}) + return nil + }) + + // run the import + err = encoder(resources) + if err != nil { + return + } + + // Adjust name res. tr. since we're changing it + if err = updateTranslations(ctx, svc.ac, svc.locale, &locale.ResourceTranslation{ + Resource: auxNs.ResourceTranslation(), + Key: types.LocaleKeyNamespaceName.Path, + Msg: locale.SanitizeMessage(auxNs.Name), + }); err != nil { + return + } + + // Reload RBAC rules (in case import brought in something new) + rbac.Global().Reload(ctx) + if err = locale.Global().ReloadResourceTranslations(ctx); err != nil { + return + } + + // Reload workflow-triggers (in case import brought in something new) + if err = automationService.DefaultWorkflow.Load(ctx); err != nil { + // should not be a fatal error + err = nil + } + + return +} + func loadNamespace(ctx context.Context, s store.Storer, namespaceID uint64) (ns *types.Namespace, err error) { if namespaceID == 0 { return nil, ChartErrInvalidNamespaceID() diff --git a/pkg/locale/load.go b/pkg/locale/load.go index 9538d2062..1b1fd957a 100644 --- a/pkg/locale/load.go +++ b/pkg/locale/load.go @@ -195,13 +195,8 @@ func loadTranslations(lang *Language) (err error) { return } -func (svc *service) loadResourceTranslations(ctx context.Context, lang *Language, tg language.Tag) (err error) { - lang.resources, err = svc.s.TransformResource(ctx, tg) - if err != nil { - return err - } - - return +func (svc *service) LoadResourceTranslations(ctx context.Context, tg language.Tag) (out map[string]map[string]*ResourceTranslation, err error) { + return svc.s.TransformResource(ctx, tg) } // procInternal reads internal YAML translation files and converts it into diff --git a/pkg/locale/service.go b/pkg/locale/service.go index 29c100cdf..9e0b8a1af 100644 --- a/pkg/locale/service.go +++ b/pkg/locale/service.go @@ -283,7 +283,7 @@ func (svc *service) ReloadResourceTranslations(ctx context.Context) (err error) svc.set[tag] = lang } - if err = svc.loadResourceTranslations(ctx, lang, lang.Tag); err != nil { + if lang.resources, err = svc.LoadResourceTranslations(ctx, lang.Tag); err != nil { return err }