From 1e6cbf6123feca1443ed999120bf026f94f58827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Wed, 1 Mar 2023 14:07:13 +0100 Subject: [PATCH] Add logic for record datasourceimport/export --- .../templates/gocode/envoy/utils.go.tpl | 1 - server/compose/envoy/record_datasource.go | 4 + server/compose/envoy/store_decode.go | 6 + server/pkg/envoyx/csv/decoder.go | 5 + server/pkg/envoyx/datasource.go | 1 + server/pkg/envoyx/envoy.go | 18 +- server/pkg/envoyx/envoy_test.go | 4 +- server/pkg/envoyx/io.go | 55 ++++- server/pkg/envoyx/json/decoder.go | 12 +- server/pkg/envoyx/json/decoder_test.go | 10 +- server/pkg/envoyx/utils.gen.go | 3 +- server/tests/envoy/import_export_test.go | 12 +- server/tests/envoy/main_test.go | 19 ++ server/tests/envoy/rbac_rules_test.go | 12 +- server/tests/envoy/records_test.go | 232 ++++++++++++++++++ server/tests/envoy/res_tr_test.go | 12 +- .../testdata/datasource_records/def.yaml | 22 ++ .../testdata/datasource_records/records.csv | 9 + 18 files changed, 393 insertions(+), 44 deletions(-) create mode 100644 server/tests/envoy/records_test.go create mode 100644 server/tests/envoy/testdata/datasource_records/def.yaml create mode 100644 server/tests/envoy/testdata/datasource_records/records.csv diff --git a/server/codegen/assets/templates/gocode/envoy/utils.go.tpl b/server/codegen/assets/templates/gocode/envoy/utils.go.tpl index 5503fddc5..c91e5f4f6 100644 --- a/server/codegen/assets/templates/gocode/envoy/utils.go.tpl +++ b/server/codegen/assets/templates/gocode/envoy/utils.go.tpl @@ -29,6 +29,5 @@ var ( // pass removes everything superNeedyResources = map[string]bool{ "corteza::compose:module-field": true, - "corteza::compose:record-datasource": true, } ) diff --git a/server/compose/envoy/record_datasource.go b/server/compose/envoy/record_datasource.go index 5a755bb5b..8b0d74f42 100644 --- a/server/compose/envoy/record_datasource.go +++ b/server/compose/envoy/record_datasource.go @@ -181,3 +181,7 @@ func (ip *iteratorProvider) Reset(ctx context.Context) (err error) { func (ip *iteratorProvider) Ident() (out string) { return } + +// @todo consider omitting these from the interface since they're not always needed +func (ip *iteratorProvider) SetIdent(string) { +} diff --git a/server/compose/envoy/store_decode.go b/server/compose/envoy/store_decode.go index 54c61e4a4..77116cf28 100644 --- a/server/compose/envoy/store_decode.go +++ b/server/compose/envoy/store_decode.go @@ -189,6 +189,7 @@ func (d StoreDecoder) decodePageRefs(p *types.Page) (refs map[string]envoyx.Ref) func (d StoreDecoder) extendDecoder(ctx context.Context, s store.Storer, dl dal.FullService, rt string, refs map[string]*envoyx.Node, rf envoyx.ResourceFilter) (out envoyx.NodeSet, err error) { switch rt { + // @todo consider hooking into the regular record resource type as well case ComposeRecordDatasourceAuxType: return d.decodeRecordDatasource(ctx, s, dl, refs, rf) } @@ -237,6 +238,11 @@ func (d StoreDecoder) decodeRecordDatasource(ctx context.Context, s store.Storer ou := &RecordDatasource{ provider: &iteratorProvider{iter: iter}, refToID: make(map[string]uint64), + // @todo consider providing defaults from the outside + mapping: datasourceMapping{ + KeyField: []string{"id"}, + Defaultable: true, + }, } rr := map[string]envoyx.Ref{ diff --git a/server/pkg/envoyx/csv/decoder.go b/server/pkg/envoyx/csv/decoder.go index fa0b207d0..a600303cb 100644 --- a/server/pkg/envoyx/csv/decoder.go +++ b/server/pkg/envoyx/csv/decoder.go @@ -93,6 +93,11 @@ func (d *decoder) Cleanup() error { return os.Remove(d.src.Name()) } +// SetIdent overwrites the system defined identifier +func (d *decoder) SetIdent(ident string) { + d.ident = ident +} + // Ident returns the assigned identifier func (d *decoder) Ident() string { return d.ident diff --git a/server/pkg/envoyx/datasource.go b/server/pkg/envoyx/datasource.go index fe6d25b1e..3aae7864d 100644 --- a/server/pkg/envoyx/datasource.go +++ b/server/pkg/envoyx/datasource.go @@ -6,6 +6,7 @@ type ( Provider interface { Next(ctx context.Context, out map[string]string) (more bool, err error) Reset(ctx context.Context) error + SetIdent(string) Ident() string } diff --git a/server/pkg/envoyx/envoy.go b/server/pkg/envoyx/envoy.go index 1f0e4f780..7e60f56e1 100644 --- a/server/pkg/envoyx/envoy.go +++ b/server/pkg/envoyx/envoy.go @@ -3,6 +3,7 @@ package envoyx import ( "context" "fmt" + "os" "strings" "github.com/cortezaproject/corteza/server/pkg/expr" @@ -57,6 +58,10 @@ type ( Decode(ctx context.Context, p DecodeParams) (out NodeSet, err error) } + canCheckFile interface { + CanFile(f *os.File) bool + } + DecodeParams struct { Type decodeType Params map[string]any @@ -131,7 +136,7 @@ func Global() *Service { } // Decode returns a set of envoy Nodes based on the given decode params -func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err error) { +func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) { err = p.validate() if err != nil { return @@ -141,7 +146,8 @@ func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err case DecodeTypeURI: return svc.decodeUri(ctx, p) case DecodeTypeStore: - return svc.decodeStore(ctx, p) + nodes, err = svc.decodeStore(ctx, p) + return default: err = fmt.Errorf("unsupported decoder type %s", p.Type) } @@ -149,7 +155,7 @@ func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err return } -func (svc *Service) Bake(ctx context.Context, p EncodeParams, nodes ...*Node) (gg *DepGraph, err error) { +func (svc *Service) Bake(ctx context.Context, p EncodeParams, providers []Provider, nodes ...*Node) (gg *DepGraph, err error) { err = svc.bakeEnvoyConfig(p.Envoy, nodes...) if err != nil { return @@ -160,6 +166,8 @@ func (svc *Service) Bake(ctx context.Context, p EncodeParams, nodes ...*Node) (g return } + svc.bakeProviders(providers, nodes...) + gg = BuildDepGraph(nodes...) return } @@ -282,6 +290,10 @@ func (svc *Service) bakeExpressions(nodes ...*Node) (err error) { return } +func (svc *Service) bakeProviders(providers []Provider, nodes ...*Node) { + SetDecoderSources(nodes, providers...) +} + func (svc *Service) mergeEnvoyConfigs(a, b EnvoyConfig) (c EnvoyConfig) { c = a if c.MergeAlg == OnConflictDefault { diff --git a/server/pkg/envoyx/envoy_test.go b/server/pkg/envoyx/envoy_test.go index 6f600f348..4bd368301 100644 --- a/server/pkg/envoyx/envoy_test.go +++ b/server/pkg/envoyx/envoy_test.go @@ -30,7 +30,7 @@ func TestBake(t *testing.T) { SkipIf: "true", }, } - _, err := (&Service{}).Bake(ctx, EncodeParams{Envoy: EnvoyConfig{MergeAlg: OnConflictPanic, SkipIf: "false"}}, a, b, c) + _, err := (&Service{}).Bake(ctx, EncodeParams{Envoy: EnvoyConfig{MergeAlg: OnConflictPanic, SkipIf: "false"}}, nil, a, b, c) req.NoError(err) req.Equal(OnConflictPanic, a.Config.MergeAlg) @@ -57,7 +57,7 @@ func TestBake(t *testing.T) { }, } - _, err := (&Service{}).Bake(ctx, EncodeParams{}, a, b) + _, err := (&Service{}).Bake(ctx, EncodeParams{}, nil, a, b) req.NoError(err) req.Nil(a.Config.SkipIfEval) diff --git a/server/pkg/envoyx/io.go b/server/pkg/envoyx/io.go index ad72bc679..f7ea6a5c8 100644 --- a/server/pkg/envoyx/io.go +++ b/server/pkg/envoyx/io.go @@ -7,9 +7,12 @@ import ( "os" "path/filepath" "strings" + + "github.com/cortezaproject/corteza/server/pkg/envoyx/csv" + "github.com/cortezaproject/corteza/server/pkg/envoyx/json" ) -func (nvyx *Service) decodeUri(ctx context.Context, p DecodeParams) (nn NodeSet, err error) { +func (svc *Service) decodeUri(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) { aUri, ok := p.Params["uri"] if !ok { err = fmt.Errorf("cannot decode URI: no uri parameter provided") @@ -31,16 +34,19 @@ func (nvyx *Service) decodeUri(ctx context.Context, p DecodeParams) (nn NodeSet, case "file": fileInfo, err := os.Stat(rest) if err != nil { - return nil, err + return nil, nil, err } if fileInfo.IsDir() { // decode whole directory - return nvyx.decodeDirectory(ctx, p, rest) + return svc.decodeDirectory(ctx, p, rest) } // decode specific file - return nvyx.decodeFile(ctx, p, rest) + var base string + bits := strings.Split(rest, string(os.PathSeparator)) + base = strings.Join(bits[0:len(bits)-1], string(os.PathSeparator)) + return svc.decodeFile(ctx, p, base, rest) default: err = fmt.Errorf("unsupported URI protocol %s", proto) @@ -61,8 +67,9 @@ func (nvyx *Service) encodeIo(ctx context.Context, dg *DepGraph, p EncodeParams) return } -func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path string) (nn NodeSet, err error) { - return nn, filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { +func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path string) (nodes NodeSet, providers []Provider, err error) { + basePath := path + return nodes, providers, filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } @@ -72,17 +79,18 @@ func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path s return nil } - aux, err := nvyx.decodeFile(ctx, p, path) + auxNodes, auxProviders, err := nvyx.decodeFile(ctx, p, basePath, path) if err != nil { return err } - nn = append(nn, aux...) + nodes = append(nodes, auxNodes...) + providers = append(providers, auxProviders...) return nil }) } -func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, path string) (nn NodeSet, err error) { +func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, basePath, path string) (nodes NodeSet, providers []Provider, err error) { var aux NodeSet f, err := os.Open(path) if err != nil { @@ -93,11 +101,17 @@ func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, path string p.Params["stream"] = f for _, d := range nvyx.decoders[DecodeTypeURI] { + if cc, ok := d.(canCheckFile); ok { + if !cc.CanFile(f) { + continue + } + } + aux, err = d.Decode(ctx, p) if err != nil { return } - nn = append(nn, aux...) + nodes = append(nodes, aux...) _, err = f.Seek(0, 0) if err != nil { @@ -105,5 +119,26 @@ func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, path string } } + providerIdent := strings.Replace(f.Name(), basePath, "", -1) + providerIdent = strings.TrimLeft(providerIdent, "/") + + if csv.CanDecodeFile(f) || csv.CanDecodeExt(f.Name()) { + f.Seek(0, 0) + aux, err := csv.Decoder(f, providerIdent) + if err != nil { + return nil, nil, err + } + providers = append(providers, aux) + } + + if json.CanDecodeFile(f) || json.CanDecodeExt(f.Name()) { + f.Seek(0, 0) + aux, err := json.Decoder(f, providerIdent) + if err != nil { + return nil, nil, err + } + providers = append(providers, aux) + } + return } diff --git a/server/pkg/envoyx/json/decoder.go b/server/pkg/envoyx/json/decoder.go index f4441f001..1ccdfaba5 100644 --- a/server/pkg/envoyx/json/decoder.go +++ b/server/pkg/envoyx/json/decoder.go @@ -1,6 +1,7 @@ -package csv +package json import ( + "context" "encoding/json" "io" "io/ioutil" @@ -94,6 +95,11 @@ func (d *decoder) Cleanup() error { return os.Remove(d.src.Name()) } +// SetIdent overwrites the system defined identifier +func (d *decoder) SetIdent(ident string) { + d.ident = ident +} + // Ident returns the assigned identifier func (d *decoder) Ident() string { return d.ident @@ -105,13 +111,13 @@ func (d *decoder) Fields() []string { } // Reset resets the decoder to the start -func (d *decoder) Reset() error { +func (d *decoder) Reset(_ context.Context) error { _, err := d.src.Seek(0, 0) return err } // Next returns the field: value mapping for the next row -func (d *decoder) Next(out map[string]string) (more bool, err error) { +func (d *decoder) Next(_ context.Context, out map[string]string) (more bool, err error) { err = d.reader.Decode(&out) if err == io.EOF { return false, nil diff --git a/server/pkg/envoyx/json/decoder_test.go b/server/pkg/envoyx/json/decoder_test.go index 374032036..23d510e77 100644 --- a/server/pkg/envoyx/json/decoder_test.go +++ b/server/pkg/envoyx/json/decoder_test.go @@ -1,4 +1,4 @@ -package csv +package json import ( "io" @@ -35,28 +35,28 @@ func TestDecoder(t *testing.T) { aux := make(map[string]string) var more bool - more, err = dc.Next(aux) + more, err = dc.Next(nil, aux) req.NoError(err) req.True(more) req.Equal("r1f1", aux["f1"]) req.Equal("r1f2", aux["f2"]) req.Equal("r1f3", aux["f3"]) - more, err = dc.Next(aux) + more, err = dc.Next(nil, aux) req.NoError(err) req.True(more) req.Equal("r2f1", aux["f1"]) req.Equal("r2f2", aux["f2"]) req.Equal("r2f3", aux["f3"]) - more, err = dc.Next(aux) + more, err = dc.Next(nil, aux) req.NoError(err) req.True(more) req.Equal("r3f1", aux["f1"]) req.Equal("r3f2", aux["f2"]) req.Equal("r3f3", aux["f3"]) - more, err = dc.Next(aux) + more, err = dc.Next(nil, aux) req.NoError(err) req.False(more) }) diff --git a/server/pkg/envoyx/utils.gen.go b/server/pkg/envoyx/utils.gen.go index e16706c0f..dc9e4e620 100644 --- a/server/pkg/envoyx/utils.gen.go +++ b/server/pkg/envoyx/utils.gen.go @@ -28,7 +28,6 @@ var ( // superNeedyResources is the second level of filtering in case the first // pass removes everything superNeedyResources = map[string]bool{ - "corteza::compose:module-field": true, - "corteza::compose:record-datasource": true, + "corteza::compose:module-field": true, } ) diff --git a/server/tests/envoy/import_export_test.go b/server/tests/envoy/import_export_test.go index ad598bdb5..64e4721ad 100644 --- a/server/tests/envoy/import_export_test.go +++ b/server/tests/envoy/import_export_test.go @@ -44,7 +44,7 @@ func TestImportExport(t *testing.T) { t.Run("initial import", func(t *testing.T) { t.Run("parse configs", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeURI, Params: map[string]any{ "uri": "file://testdata/full", @@ -60,7 +60,7 @@ func TestImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) @@ -87,7 +87,7 @@ func TestImportExport(t *testing.T) { t.Run("export", func(t *testing.T) { t.Run("export from DB", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeStore, Params: map[string]any{ "storer": defaultStore, @@ -122,7 +122,7 @@ func TestImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) @@ -141,7 +141,7 @@ func TestImportExport(t *testing.T) { t.Run("second import", func(t *testing.T) { t.Run("yaml parse", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeURI, Params: map[string]any{ "uri": fmt.Sprintf("file://%s", auxFile.Name()), @@ -157,7 +157,7 @@ func TestImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) diff --git a/server/tests/envoy/main_test.go b/server/tests/envoy/main_test.go index b83ad05a5..a9edbcecc 100644 --- a/server/tests/envoy/main_test.go +++ b/server/tests/envoy/main_test.go @@ -130,12 +130,28 @@ func cleanup(t *testing.T) { store.TruncateAutomationWorkflows(ctx, defaultStore), store.TruncateAutomationTriggers(ctx, defaultStore), + + truncateRecords(ctx), ) if err != nil { t.Fatalf("failed to decode scenario data: %v", err) } } +func truncateRecords(ctx context.Context) error { + models, err := defaultDal.SearchModels(ctx) + if err != nil { + return err + } + for _, model := range models { + err = defaultDal.Truncate(ctx, model.ToFilter(), nil) + if err != nil { + return err + } + } + return nil +} + func collect(ee ...error) error { for _, e := range ee { if e != nil { @@ -168,4 +184,7 @@ func initSvc(ctx context.Context) { systemEnvoy.StoreEncoder{}, automationEnvoy.StoreEncoder{}, ) + defaultEnvoy.AddEncoder(envoyx.EncodeTypeIo, + composeEnvoy.CsvEncoder{}, + ) } diff --git a/server/tests/envoy/rbac_rules_test.go b/server/tests/envoy/rbac_rules_test.go index 3cb0c945e..d2006f149 100644 --- a/server/tests/envoy/rbac_rules_test.go +++ b/server/tests/envoy/rbac_rules_test.go @@ -43,7 +43,7 @@ func TestRbacImportExport(t *testing.T) { t.Run("initial import", func(t *testing.T) { t.Run("parse configs", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeURI, Params: map[string]any{ "uri": "file://testdata/rbac", @@ -59,7 +59,7 @@ func TestRbacImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) @@ -87,7 +87,7 @@ func TestRbacImportExport(t *testing.T) { var rules envoyx.NodeSet t.Run("export", func(t *testing.T) { t.Run("export from DB", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeStore, Params: map[string]any{ "storer": defaultStore, @@ -122,7 +122,7 @@ func TestRbacImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) @@ -141,7 +141,7 @@ func TestRbacImportExport(t *testing.T) { t.Run("second import", func(t *testing.T) { t.Run("yaml parse", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeURI, Params: map[string]any{ "uri": fmt.Sprintf("file://%s", auxFile.Name()), @@ -157,7 +157,7 @@ func TestRbacImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) diff --git a/server/tests/envoy/records_test.go b/server/tests/envoy/records_test.go new file mode 100644 index 000000000..4e73fe363 --- /dev/null +++ b/server/tests/envoy/records_test.go @@ -0,0 +1,232 @@ +package envoy + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/cortezaproject/corteza/server/compose/dalutils" + "github.com/cortezaproject/corteza/server/compose/envoy" + "github.com/cortezaproject/corteza/server/compose/types" + "github.com/cortezaproject/corteza/server/pkg/dal" + "github.com/cortezaproject/corteza/server/pkg/envoyx" + "github.com/cortezaproject/corteza/server/store" + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" +) + +func TestRecordsImportExport(t *testing.T) { + var ( + ctx = context.Background() + req = require.New(t) + nodes envoyx.NodeSet + providers []envoyx.Provider + gg *envoyx.DepGraph + err error + ) + _ = gg + + cleanup(t) + + // The test + // + // * imports some YAML files + // * checks the DB state + // * exports the DB into a YAML + // * clears the DB + // * imports the YAML + // * checks the DB state the same way as before + // + // The above outlined flow allows us to trivially check if the data is both + // imported and exported correctly. + // + // The initial step could also manually populate the DB but the YAML import + // is more convenient. + + t.Run("initial import", func(t *testing.T) { + t.Run("parse configs", func(t *testing.T) { + nodes, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + Type: envoyx.DecodeTypeURI, + Params: map[string]any{ + "uri": "file://testdata/datasource_records", + }, + }) + req.NoError(err) + }) + + t.Run("bake", func(t *testing.T) { + gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, providers, nodes...) + req.NoError(err) + }) + + t.Run("import into DB", func(t *testing.T) { + err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, gg) + req.NoError(err) + }) + + assertRecordState(ctx, t, defaultStore, defaultDal, req) + }) + + // Prepare a temp file where we'll dump the YAML into + auxFile, err := os.CreateTemp(os.TempDir(), "*.csv") + req.NoError(err) + spew.Dump(auxFile.Name()) + // defer os.Remove(auxFile.Name()) + defer auxFile.Close() + + t.Run("export", func(t *testing.T) { + t.Run("export from DB", func(t *testing.T) { + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + Type: envoyx.DecodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + Filter: map[string]envoyx.ResourceFilter{ + envoy.ComposeRecordDatasourceAuxType: { + Refs: map[string]envoyx.Ref{ + "NamespaceID": { + ResourceType: types.NamespaceResourceType, + Identifiers: envoyx.MakeIdentifiers("test_ns_1"), + Scope: envoyx.Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: envoyx.MakeIdentifiers("test_ns_1")}, + }, + "ModuleID": { + ResourceType: types.ModuleResourceType, + Identifiers: envoyx.MakeIdentifiers("test_ns_1_mod_1"), + Scope: envoyx.Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: envoyx.MakeIdentifiers("test_ns_1"), + }, + }, + }, + Scope: envoyx.Scope{ + ResourceType: types.NamespaceResourceType, + Identifiers: envoyx.MakeIdentifiers("test_ns_1"), + }, + }, + }, + }) + req.NoError(err) + }) + + t.Run("bake", func(t *testing.T) { + gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, nil, nodes...) + req.NoError(err) + }) + + t.Run("write file", func(t *testing.T) { + err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeIo, + Params: map[string]any{ + "writer": auxFile, + }, + }, gg) + req.NoError(err) + }) + }) + + cleanup(t) + + t.Run("second import", func(t *testing.T) { + t.Run("yaml parse", func(t *testing.T) { + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + Type: envoyx.DecodeTypeURI, + Params: map[string]any{ + "uri": "file://testdata/datasource_records", + }, + }) + req.NoError(err) + + _, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + Type: envoyx.DecodeTypeURI, + Params: map[string]any{ + "uri": fmt.Sprintf("file://%s", auxFile.Name()), + }, + }) + req.NoError(err) + for _, p := range providers { + p.SetIdent("records.csv") + } + }) + + t.Run("bake", func(t *testing.T) { + gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, providers, nodes...) + req.NoError(err) + }) + + t.Run("run import", func(t *testing.T) { + err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{ + Type: envoyx.EncodeTypeStore, + Params: map[string]any{ + "storer": defaultStore, + "dal": defaultDal, + }, + }, gg) + req.NoError(err) + }) + + assertRecordState(ctx, t, defaultStore, defaultDal, req) + }) +} + +func assertRecordState(ctx context.Context, t *testing.T, s store.Storer, dl dal.FullService, req *require.Assertions) { + t.Run("check state", func(t *testing.T) { + ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns_1") + req.NoError(err) + + mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "test_ns_1_mod_1") + req.NoError(err) + + rr, _, err := dalutils.ComposeRecordsList(ctx, dl, mod, types.RecordFilter{}) + req.NoError(err) + + req.Len(rr, 8) + + compareValues(req, rr[0].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_1_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_1_src_c2"}}) + compareValues(req, rr[1].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_2_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_2_src_c2"}}) + compareValues(req, rr[2].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_3_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_3_src_c2"}}) + compareValues(req, rr[3].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_4_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_4_src_c2"}}) + compareValues(req, rr[4].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_5_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_5_src_c2"}}) + compareValues(req, rr[5].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_6_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_6_src_c2"}}) + compareValues(req, rr[6].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_7_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_7_src_c2"}}) + compareValues(req, rr[7].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_8_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_8_src_c2"}}) + + }) +} + +func compareValues(req *require.Assertions, a, b types.RecordValueSet) { + for _, va := range a { + vb := b.Get(va.Name, va.Place) + req.NotNil(vb) + + req.Equal(va.Value, vb.Value) + } + +} diff --git a/server/tests/envoy/res_tr_test.go b/server/tests/envoy/res_tr_test.go index fcc287584..4ab16e8b3 100644 --- a/server/tests/envoy/res_tr_test.go +++ b/server/tests/envoy/res_tr_test.go @@ -43,7 +43,7 @@ func TestResTrImportExport(t *testing.T) { t.Run("initial import", func(t *testing.T) { t.Run("parse configs", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeURI, Params: map[string]any{ "uri": "file://testdata/locale", @@ -59,7 +59,7 @@ func TestResTrImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) @@ -88,7 +88,7 @@ func TestResTrImportExport(t *testing.T) { var translations envoyx.NodeSet t.Run("export", func(t *testing.T) { t.Run("export from DB", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeStore, Params: map[string]any{ "storer": defaultStore, @@ -122,7 +122,7 @@ func TestResTrImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) @@ -141,7 +141,7 @@ func TestResTrImportExport(t *testing.T) { t.Run("second import", func(t *testing.T) { t.Run("yaml parse", func(t *testing.T) { - nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ + nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{ Type: envoyx.DecodeTypeURI, Params: map[string]any{ "uri": fmt.Sprintf("file://%s", auxFile.Name()), @@ -157,7 +157,7 @@ func TestResTrImportExport(t *testing.T) { "storer": defaultStore, "dal": defaultDal, }, - }, nodes...) + }, nil, nodes...) req.NoError(err) }) diff --git a/server/tests/envoy/testdata/datasource_records/def.yaml b/server/tests/envoy/testdata/datasource_records/def.yaml new file mode 100644 index 000000000..9197302a9 --- /dev/null +++ b/server/tests/envoy/testdata/datasource_records/def.yaml @@ -0,0 +1,22 @@ +namespace: + test_ns_1: + name: Test Namespace 1 + modules: + test_ns_1_mod_1: + name: Test Namespace 1 Module 1 + fields: + test_ns_1_mod_1_f1: + title: Test Namespace 1 Module 1 Field 1 + test_ns_1_mod_1_f2: + title: Test Namespace 1 Module 1 Field 2 + kind: Number + source: + - from: records.csv + key: ID + map: + - column: ID + skip: true + - column: test_ns_1_mod_1_f1 + field: test_ns_1_mod_1_f1 + - column: test_ns_1_mod_1_f2 + field: test_ns_1_mod_1_f2 diff --git a/server/tests/envoy/testdata/datasource_records/records.csv b/server/tests/envoy/testdata/datasource_records/records.csv new file mode 100644 index 000000000..5bbff703b --- /dev/null +++ b/server/tests/envoy/testdata/datasource_records/records.csv @@ -0,0 +1,9 @@ +ID,test_ns_1_mod_1_f1,test_ns_1_mod_1_f2 +row_1,row_1_src_c1,row_1_src_c2 +row_2,row_2_src_c1,row_2_src_c2 +row_3,row_3_src_c1,row_3_src_c2 +row_4,row_4_src_c1,row_4_src_c2 +row_5,row_5_src_c1,row_5_src_c2 +row_6,row_6_src_c1,row_6_src_c2 +row_7,row_7_src_c1,row_7_src_c2 +row_8,row_8_src_c1,row_8_src_c2