DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Tweak Envoy store -> yaml

Browse Source 
* write to fs,
* temporary remove compact format,
* make complete namespace exports easier.
This commit is contained in:
Tomaž Jerman
2021-03-30 09:25:09 +02:00
parent eaa6d8f75f
commit ed8fddd417
7 changed files with 105 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
pkg/envoy/store/compose.go
View File

@@ -32,13 +32,15 @@ type (
}
composeDecoder struct {
resourceID []uint64
resourceID []uint64
namespaceID []uint64
}
)
func newComposeDecoder() *composeDecoder {
return &composeDecoder{
resourceID: make([]uint64, 0, 200),
resourceID: make([]uint64, 0, 200),
namespaceID: make([]uint64, 0, 200),
}
}
@@ -69,6 +71,7 @@ func (d *composeDecoder) decodeComposeNamespace(ctx context.Context, s composeSt
}
for _, n := range nn {
d.namespaceID = append(d.namespaceID, n.ID)
d.resourceID = append(d.resourceID, n.ID)
mm = append(mm, newComposeNamespace(n))
@@ -95,6 +98,18 @@ func (d *composeDecoder) decodeComposeModule(ctx context.Context, s composeStore
}
}
if len(d.namespaceID) > 0 {
ffNs := make([]*composeModuleFilter, 0, len(ff)+len(d.namespaceID))
for _, nsID := range d.namespaceID {
for _, f := range ff {
fNs := *f
fNs.NamespaceID = nsID
ffNs = append(ffNs, &fNs)
}
}
ff = ffNs
}
var nn types.ModuleSet
var fn types.ModuleFilter
var err error
@@ -150,6 +165,18 @@ func (d *composeDecoder) decodeComposeRecord(ctx context.Context, s store.Storer
}
}
if len(d.namespaceID) > 0 {
ffNs := make([]*composeRecordFilter, 0, len(ff)+len(d.namespaceID))
for _, nsID := range d.namespaceID {
for _, f := range ff {
fNs := *f
fNs.NamespaceID = nsID
ffNs = append(ffNs, &fNs)
}
}
ff = ffNs
}
// When decoding large amounts of records (milions) we can probably assume
// that each system user exists somewhere in the record set.
//
@@ -265,6 +292,18 @@ func (d *composeDecoder) decodeComposePage(ctx context.Context, s composeStore,
}
}
if len(d.namespaceID) > 0 {
ffNs := make([]*composePageFilter, 0, len(ff)+len(d.namespaceID))
for _, nsID := range d.namespaceID {
for _, f := range ff {
fNs := *f
fNs.NamespaceID = nsID
ffNs = append(ffNs, &fNs)
}
}
ff = ffNs
}
var nn types.PageSet
var fn types.PageFilter
var err error
@@ -310,6 +349,18 @@ func (d *composeDecoder) decodeComposeChart(ctx context.Context, s composeStore,
}
}
if len(d.namespaceID) > 0 {
ffNs := make([]*composeChartFilter, 0, len(ff)+len(d.namespaceID))
for _, nsID := range d.namespaceID {
for _, f := range ff {
fNs := *f
fNs.NamespaceID = nsID
ffNs = append(ffNs, &fNs)
}
}
ff = ffNs
}
var nn types.ChartSet
var fn types.ChartFilter
var err error

10
pkg/envoy/store/system.go
View File

@@ -328,15 +328,15 @@ func (df *DecodeFilter) systemFromResource(rr ...string) *DecodeFilter {
}
switch strings.ToLower(r) {
case "system:roles":
case "system:role":
df = df.Roles(&types.RoleFilter{
Query: id,
})
case "system:users":
case "system:user":
df = df.Users(&types.UserFilter{
Query: id,
})
case "system:templates":
case "system:template":
df = df.Templates(&types.TemplateFilter{
Handle: id,
})
@@ -346,11 +346,11 @@ func (df *DecodeFilter) systemFromResource(rr ...string) *DecodeFilter {
TemplateID: []uint64{templateID},
})
}
case "system:applications":
case "system:application":
df = df.Applications(&types.ApplicationFilter{
Query: id,
})
case "system:settins":
case "system:setting":
df = df.Settings(&types.SettingsFilter{})
case "system:rbac":
df = df.Rbac(&rbac.RuleFilter{})

11
pkg/envoy/yaml/compose_chart_marshal.go
View File

@@ -71,11 +71,12 @@ func (n *composeChart) Encode(ctx context.Context, doc *Document, state *envoy.R
// @todo skip eval?
if n.encoderConfig.CompactOutput {
err = doc.nestComposeChart(n.refNamespace, n)
} else {
doc.addComposeChart(n)
}
// if n.encoderConfig.CompactOutput {
// err = doc.nestComposeChart(n.refNamespace, n)
// } else {
// doc.addComposeChart(n)
// }
doc.addComposeChart(n)
return err
}

11
pkg/envoy/yaml/compose_module_marshal.go
View File

@@ -85,11 +85,12 @@ func (n *composeModule) Encode(ctx context.Context, doc *Document, state *envoy.
// @todo skip eval?
if n.encoderConfig.CompactOutput {
err = doc.nestComposeModule(n.refNamespace, n)
} else {
doc.addComposeModule(n)
}
// if n.encoderConfig.CompactOutput {
// err = doc.nestComposeModule(n.refNamespace, n)
// } else {
// doc.addComposeModule(n)
// }
doc.addComposeModule(n)
return err
}

19
pkg/envoy/yaml/compose_page_marshal.go
View File

@@ -101,15 +101,16 @@ func (n *composePage) Encode(ctx context.Context, doc *Document, state *envoy.Re
// @todo skip eval?
if n.encoderConfig.CompactOutput {
if n.refParent != "" {
err = doc.nestComposePageChild(n.refParent, n)
} else {
err = doc.nestComposePage(n.refNamespace, n)
}
} else {
doc.addComposePage(n)
}
// if n.encoderConfig.CompactOutput {
// if n.refParent != "" {
// err = doc.nestComposePageChild(n.refParent, n)
// } else {
// err = doc.nestComposePage(n.refNamespace, n)
// }
// } else {
// doc.addComposePage(n)
// }
doc.addComposePage(n)
return err
}

4
pkg/envoy/yaml/encoder.go
View File

@@ -58,8 +58,8 @@ type (
// If not defined, RFC3339 is used (this one - 2006-01-02T15:04:05Z07:00)
TimeLayout string
// CompactOutput forces the output to be as compact as possible
CompactOutput bool
// // CompactOutput forces the output to be as compact as possible
// CompactOutput bool
// MappedOutput forces the sequences to encode as maps (where possible)
MappedOutput bool

31
system/commands/exporter.go
View File

@@ -3,6 +3,9 @@ package commands
import (
"context"
"io"
"os"
"path"
"strings"
"github.com/cortezaproject/corteza-server/pkg/envoy/yaml"
"github.com/spf13/cobra"
@@ -15,6 +18,10 @@ import (
)
func Export(storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Command {
var (
output string
)
cmd := &cobra.Command{
Use: "export",
Short: "Export",
@@ -32,15 +39,13 @@ func Export(storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Co
f = f.FromResource(args...)
// Nothing to do here...
sd := su.Decoder()
nn, err := sd.Decode(ctx, s, f)
cli.HandleError(err)
ye := yaml.NewYamlEncoder(&yaml.EncoderConfig{
MappedOutput: true,
CompactOutput: true,
MappedOutput: true,
// CompactOutput: true,
})
bld := envoy.NewBuilder(ye)
g, err := bld.Build(ctx, nn...)
@@ -48,15 +53,25 @@ func Export(storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Co
err = envoy.Encode(ctx, g, ye)
cli.HandleError(err)
ss := ye.Stream()
_ = ss
makeFN := func(base, res string) string {
pp := strings.Split(strings.Trim(res, ":"), ":")
name := strings.Join(pp, "_") + ".yaml"
return path.Join(base, name)
}
// Std out
// @todo write to directory?
w := cmd.OutOrStdout()
for _, s := range ss {
io.Copy(w, s.Source)
f, err := os.Create(makeFN(output, s.Resource))
cli.HandleError(err)
defer f.Close()
io.Copy(f, s.Source)
}
},
}
cmd.Flags().StringVarP(&output, "out", "o", "./", "The directory to write output files to")
return cmd
}