From ed8fddd417931448eb8942ac79affdd14cdbddea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 30 Mar 2021 09:25:09 +0200 Subject: [PATCH] Tweak Envoy store -> yaml * write to fs, * temporary remove compact format, * make complete namespace exports easier. --- pkg/envoy/store/compose.go | 55 +++++++++++++++++++++++- pkg/envoy/store/system.go | 10 ++--- pkg/envoy/yaml/compose_chart_marshal.go | 11 ++--- pkg/envoy/yaml/compose_module_marshal.go | 11 ++--- pkg/envoy/yaml/compose_page_marshal.go | 19 ++++---- pkg/envoy/yaml/encoder.go | 4 +- system/commands/exporter.go | 31 +++++++++---- 7 files changed, 105 insertions(+), 36 deletions(-) diff --git a/pkg/envoy/store/compose.go b/pkg/envoy/store/compose.go index ab60c7b62..d4268a1c5 100644 --- a/pkg/envoy/store/compose.go +++ b/pkg/envoy/store/compose.go @@ -32,13 +32,15 @@ type ( } composeDecoder struct { - resourceID []uint64 + resourceID []uint64 + namespaceID []uint64 } ) func newComposeDecoder() *composeDecoder { return &composeDecoder{ - resourceID: make([]uint64, 0, 200), + resourceID: make([]uint64, 0, 200), + namespaceID: make([]uint64, 0, 200), } } @@ -69,6 +71,7 @@ func (d *composeDecoder) decodeComposeNamespace(ctx context.Context, s composeSt } for _, n := range nn { + d.namespaceID = append(d.namespaceID, n.ID) d.resourceID = append(d.resourceID, n.ID) mm = append(mm, newComposeNamespace(n)) @@ -95,6 +98,18 @@ func (d *composeDecoder) decodeComposeModule(ctx context.Context, s composeStore } } + if len(d.namespaceID) > 0 { + ffNs := make([]*composeModuleFilter, 0, len(ff)+len(d.namespaceID)) + for _, nsID := range d.namespaceID { + for _, f := range ff { + fNs := *f + fNs.NamespaceID = nsID + ffNs = append(ffNs, &fNs) + } + } + ff = ffNs + } + var nn types.ModuleSet var fn types.ModuleFilter var err error @@ -150,6 +165,18 @@ func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer } } + if len(d.namespaceID) > 0 { + ffNs := make([]*composeRecordFilter, 0, len(ff)+len(d.namespaceID)) + for _, nsID := range d.namespaceID { + for _, f := range ff { + fNs := *f + fNs.NamespaceID = nsID + ffNs = append(ffNs, &fNs) + } + } + ff = ffNs + } + // When decoding large amounts of records (milions) we can probably assume // that each system user exists somewhere in the record set. // @@ -265,6 +292,18 @@ func (d *composeDecoder) decodeComposePage(ctx context.Context, s composeStore, } } + if len(d.namespaceID) > 0 { + ffNs := make([]*composePageFilter, 0, len(ff)+len(d.namespaceID)) + for _, nsID := range d.namespaceID { + for _, f := range ff { + fNs := *f + fNs.NamespaceID = nsID + ffNs = append(ffNs, &fNs) + } + } + ff = ffNs + } + var nn types.PageSet var fn types.PageFilter var err error @@ -310,6 +349,18 @@ func (d *composeDecoder) decodeComposeChart(ctx context.Context, s composeStore, } } + if len(d.namespaceID) > 0 { + ffNs := make([]*composeChartFilter, 0, len(ff)+len(d.namespaceID)) + for _, nsID := range d.namespaceID { + for _, f := range ff { + fNs := *f + fNs.NamespaceID = nsID + ffNs = append(ffNs, &fNs) + } + } + ff = ffNs + } + var nn types.ChartSet var fn types.ChartFilter var err error diff --git a/pkg/envoy/store/system.go b/pkg/envoy/store/system.go index 3e136e15a..b2de8e80d 100644 --- a/pkg/envoy/store/system.go +++ b/pkg/envoy/store/system.go @@ -328,15 +328,15 @@ func (df *DecodeFilter) systemFromResource(rr ...string) *DecodeFilter { } switch strings.ToLower(r) { - case "system:roles": + case "system:role": df = df.Roles(&types.RoleFilter{ Query: id, }) - case "system:users": + case "system:user": df = df.Users(&types.UserFilter{ Query: id, }) - case "system:templates": + case "system:template": df = df.Templates(&types.TemplateFilter{ Handle: id, }) @@ -346,11 +346,11 @@ func (df *DecodeFilter) systemFromResource(rr ...string) *DecodeFilter { TemplateID: []uint64{templateID}, }) } - case "system:applications": + case "system:application": df = df.Applications(&types.ApplicationFilter{ Query: id, }) - case "system:settins": + case "system:setting": df = df.Settings(&types.SettingsFilter{}) case "system:rbac": df = df.Rbac(&rbac.RuleFilter{}) diff --git a/pkg/envoy/yaml/compose_chart_marshal.go b/pkg/envoy/yaml/compose_chart_marshal.go index 01bf3361b..610121d45 100644 --- a/pkg/envoy/yaml/compose_chart_marshal.go +++ b/pkg/envoy/yaml/compose_chart_marshal.go @@ -71,11 +71,12 @@ func (n *composeChart) Encode(ctx context.Context, doc *Document, state *envoy.R // @todo skip eval? - if n.encoderConfig.CompactOutput { - err = doc.nestComposeChart(n.refNamespace, n) - } else { - doc.addComposeChart(n) - } + // if n.encoderConfig.CompactOutput { + // err = doc.nestComposeChart(n.refNamespace, n) + // } else { + // doc.addComposeChart(n) + // } + doc.addComposeChart(n) return err } diff --git a/pkg/envoy/yaml/compose_module_marshal.go b/pkg/envoy/yaml/compose_module_marshal.go index 6bd367f36..3a2d0627e 100644 --- a/pkg/envoy/yaml/compose_module_marshal.go +++ b/pkg/envoy/yaml/compose_module_marshal.go @@ -85,11 +85,12 @@ func (n *composeModule) Encode(ctx context.Context, doc *Document, state *envoy. // @todo skip eval? - if n.encoderConfig.CompactOutput { - err = doc.nestComposeModule(n.refNamespace, n) - } else { - doc.addComposeModule(n) - } + // if n.encoderConfig.CompactOutput { + // err = doc.nestComposeModule(n.refNamespace, n) + // } else { + // doc.addComposeModule(n) + // } + doc.addComposeModule(n) return err } diff --git a/pkg/envoy/yaml/compose_page_marshal.go b/pkg/envoy/yaml/compose_page_marshal.go index 88453c3b7..f8f24be46 100644 --- a/pkg/envoy/yaml/compose_page_marshal.go +++ b/pkg/envoy/yaml/compose_page_marshal.go @@ -101,15 +101,16 @@ func (n *composePage) Encode(ctx context.Context, doc *Document, state *envoy.Re // @todo skip eval? - if n.encoderConfig.CompactOutput { - if n.refParent != "" { - err = doc.nestComposePageChild(n.refParent, n) - } else { - err = doc.nestComposePage(n.refNamespace, n) - } - } else { - doc.addComposePage(n) - } + // if n.encoderConfig.CompactOutput { + // if n.refParent != "" { + // err = doc.nestComposePageChild(n.refParent, n) + // } else { + // err = doc.nestComposePage(n.refNamespace, n) + // } + // } else { + // doc.addComposePage(n) + // } + doc.addComposePage(n) return err } diff --git a/pkg/envoy/yaml/encoder.go b/pkg/envoy/yaml/encoder.go index dca5b3330..9572f2937 100644 --- a/pkg/envoy/yaml/encoder.go +++ b/pkg/envoy/yaml/encoder.go @@ -58,8 +58,8 @@ type ( // If not defined, RFC3339 is used (this one - 2006-01-02T15:04:05Z07:00) TimeLayout string - // CompactOutput forces the output to be as compact as possible - CompactOutput bool + // // CompactOutput forces the output to be as compact as possible + // CompactOutput bool // MappedOutput forces the sequences to encode as maps (where possible) MappedOutput bool diff --git a/system/commands/exporter.go b/system/commands/exporter.go index 6426562a4..faa00e626 100644 --- a/system/commands/exporter.go +++ b/system/commands/exporter.go @@ -3,6 +3,9 @@ package commands import ( "context" "io" + "os" + "path" + "strings" "github.com/cortezaproject/corteza-server/pkg/envoy/yaml" "github.com/spf13/cobra" @@ -15,6 +18,10 @@ import ( ) func Export(storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Command { + var ( + output string + ) + cmd := &cobra.Command{ Use: "export", Short: "Export", @@ -32,15 +39,13 @@ func Export(storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Co f = f.FromResource(args...) - // Nothing to do here... - sd := su.Decoder() nn, err := sd.Decode(ctx, s, f) cli.HandleError(err) ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{ - MappedOutput: true, - CompactOutput: true, + MappedOutput: true, + // CompactOutput: true, }) bld := envoy.NewBuilder(ye) g, err := bld.Build(ctx, nn...) @@ -48,15 +53,25 @@ func Export(storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Co err = envoy.Encode(ctx, g, ye) cli.HandleError(err) ss := ye.Stream() + _ = ss + + makeFN := func(base, res string) string { + pp := strings.Split(strings.Trim(res, ":"), ":") + name := strings.Join(pp, "_") + ".yaml" + return path.Join(base, name) + } - // Std out - // @todo write to directory? - w := cmd.OutOrStdout() for _, s := range ss { - io.Copy(w, s.Source) + f, err := os.Create(makeFN(output, s.Resource)) + cli.HandleError(err) + defer f.Close() + + io.Copy(f, s.Source) } }, } + cmd.Flags().StringVarP(&output, "out", "o", "./", "The directory to write output files to") + return cmd }