DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Add logic for record datasourceimport/export

Browse Source
This commit is contained in:
Tomaž Jerman
2023-03-17 10:58:46 +01:00
parent b975b2b56a
commit 1e6cbf6123
18 changed files with 393 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/codegen/assets/templates/gocode/envoy/utils.go.tpl
-1
View File
@@ -29,6 +29,5 @@ var (
// pass removes everything
superNeedyResources = map[string]bool{
"corteza::compose:module-field": true,
"corteza::compose:record-datasource": true,
}
)
server/compose/envoy/record_datasource.go
+4
View File
@@ -181,3 +181,7 @@ func (ip *iteratorProvider) Reset(ctx context.Context) (err error) {
func (ip *iteratorProvider) Ident() (out string) {
return
}
// @todo consider omitting these from the interface since they're not always needed
func (ip *iteratorProvider) SetIdent(string) {
}
server/compose/envoy/store_decode.go
+6
View File
@@ -189,6 +189,7 @@ func (d StoreDecoder) decodePageRefs(p *types.Page) (refs map[string]envoyx.Ref)
func (d StoreDecoder) extendDecoder(ctx context.Context, s store.Storer, dl dal.FullService, rt string, refs map[string]*envoyx.Node, rf envoyx.ResourceFilter) (out envoyx.NodeSet, err error) {
switch rt {
// @todo consider hooking into the regular record resource type as well
case ComposeRecordDatasourceAuxType:
return d.decodeRecordDatasource(ctx, s, dl, refs, rf)
}
@@ -237,6 +238,11 @@ func (d StoreDecoder) decodeRecordDatasource(ctx context.Context, s store.Storer
ou := &RecordDatasource{
provider: &iteratorProvider{iter: iter},
refToID: make(map[string]uint64),
// @todo consider providing defaults from the outside
mapping: datasourceMapping{
KeyField: []string{"id"},
Defaultable: true,
},
}
rr := map[string]envoyx.Ref{
server/pkg/envoyx/csv/decoder.go
+5
View File
@@ -93,6 +93,11 @@ func (d *decoder) Cleanup() error {
return os.Remove(d.src.Name())
}
// SetIdent overwrites the system defined identifier
func (d *decoder) SetIdent(ident string) {
d.ident = ident
}
// Ident returns the assigned identifier
func (d *decoder) Ident() string {
return d.ident
server/pkg/envoyx/datasource.go
+1
View File
@@ -6,6 +6,7 @@ type (
Provider interface {
Next(ctx context.Context, out map[string]string) (more bool, err error)
Reset(ctx context.Context) error
SetIdent(string)
Ident() string
}
server/pkg/envoyx/envoy.go
+15 -3
View File
@@ -3,6 +3,7 @@ package envoyx
import (
"context"
"fmt"
"os"
"strings"
"github.com/cortezaproject/corteza/server/pkg/expr"
@@ -57,6 +58,10 @@ type (
Decode(ctx context.Context, p DecodeParams) (out NodeSet, err error)
}
canCheckFile interface {
CanFile(f *os.File) bool
}
DecodeParams struct {
Type decodeType
Params map[string]any
@@ -131,7 +136,7 @@ func Global() *Service {
}
// Decode returns a set of envoy Nodes based on the given decode params
func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err error) {
func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) {
err = p.validate()
if err != nil {
return
@@ -141,7 +146,8 @@ func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err
case DecodeTypeURI:
return svc.decodeUri(ctx, p)
case DecodeTypeStore:
return svc.decodeStore(ctx, p)
nodes, err = svc.decodeStore(ctx, p)
return
default:
err = fmt.Errorf("unsupported decoder type %s", p.Type)
}
@@ -149,7 +155,7 @@ func (svc *Service) Decode(ctx context.Context, p DecodeParams) (nn NodeSet, err
return
}
func (svc *Service) Bake(ctx context.Context, p EncodeParams, nodes ...*Node) (gg *DepGraph, err error) {
func (svc *Service) Bake(ctx context.Context, p EncodeParams, providers []Provider, nodes ...*Node) (gg *DepGraph, err error) {
err = svc.bakeEnvoyConfig(p.Envoy, nodes...)
if err != nil {
return
@@ -160,6 +166,8 @@ func (svc *Service) Bake(ctx context.Context, p EncodeParams, nodes ...*Node) (g
return
}
svc.bakeProviders(providers, nodes...)
gg = BuildDepGraph(nodes...)
return
}
@@ -282,6 +290,10 @@ func (svc *Service) bakeExpressions(nodes ...*Node) (err error) {
return
}
func (svc *Service) bakeProviders(providers []Provider, nodes ...*Node) {
SetDecoderSources(nodes, providers...)
}
func (svc *Service) mergeEnvoyConfigs(a, b EnvoyConfig) (c EnvoyConfig) {
c = a
if c.MergeAlg == OnConflictDefault {
server/pkg/envoyx/envoy_test.go
+2 -2
View File
@@ -30,7 +30,7 @@ func TestBake(t *testing.T) {
SkipIf: "true",
},
}
_, err := (&Service{}).Bake(ctx, EncodeParams{Envoy: EnvoyConfig{MergeAlg: OnConflictPanic, SkipIf: "false"}}, a, b, c)
_, err := (&Service{}).Bake(ctx, EncodeParams{Envoy: EnvoyConfig{MergeAlg: OnConflictPanic, SkipIf: "false"}}, nil, a, b, c)
req.NoError(err)
req.Equal(OnConflictPanic, a.Config.MergeAlg)
@@ -57,7 +57,7 @@ func TestBake(t *testing.T) {
},
}
_, err := (&Service{}).Bake(ctx, EncodeParams{}, a, b)
_, err := (&Service{}).Bake(ctx, EncodeParams{}, nil, a, b)
req.NoError(err)
req.Nil(a.Config.SkipIfEval)
server/pkg/envoyx/io.go
+45 -10
View File
@@ -7,9 +7,12 @@ import (
"os"
"path/filepath"
"strings"
"github.com/cortezaproject/corteza/server/pkg/envoyx/csv"
"github.com/cortezaproject/corteza/server/pkg/envoyx/json"
)
func (nvyx *Service) decodeUri(ctx context.Context, p DecodeParams) (nn NodeSet, err error) {
func (svc *Service) decodeUri(ctx context.Context, p DecodeParams) (nodes NodeSet, providers []Provider, err error) {
aUri, ok := p.Params["uri"]
if !ok {
err = fmt.Errorf("cannot decode URI: no uri parameter provided")
@@ -31,16 +34,19 @@ func (nvyx *Service) decodeUri(ctx context.Context, p DecodeParams) (nn NodeSet,
case "file":
fileInfo, err := os.Stat(rest)
if err != nil {
return nil, err
return nil, nil, err
}
if fileInfo.IsDir() {
// decode whole directory
return nvyx.decodeDirectory(ctx, p, rest)
return svc.decodeDirectory(ctx, p, rest)
}
// decode specific file
return nvyx.decodeFile(ctx, p, rest)
var base string
bits := strings.Split(rest, string(os.PathSeparator))
base = strings.Join(bits[0:len(bits)-1], string(os.PathSeparator))
return svc.decodeFile(ctx, p, base, rest)
default:
err = fmt.Errorf("unsupported URI protocol %s", proto)
@@ -61,8 +67,9 @@ func (nvyx *Service) encodeIo(ctx context.Context, dg *DepGraph, p EncodeParams)
return
}
func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path string) (nn NodeSet, err error) {
return nn, filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path string) (nodes NodeSet, providers []Provider, err error) {
basePath := path
return nodes, providers, filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
@@ -72,17 +79,18 @@ func (nvyx *Service) decodeDirectory(ctx context.Context, p DecodeParams, path s
return nil
}
aux, err := nvyx.decodeFile(ctx, p, path)
auxNodes, auxProviders, err := nvyx.decodeFile(ctx, p, basePath, path)
if err != nil {
return err
}
nn = append(nn, aux...)
nodes = append(nodes, auxNodes...)
providers = append(providers, auxProviders...)
return nil
})
}
func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, path string) (nn NodeSet, err error) {
func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, basePath, path string) (nodes NodeSet, providers []Provider, err error) {
var aux NodeSet
f, err := os.Open(path)
if err != nil {
@@ -93,11 +101,17 @@ func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, path string
p.Params["stream"] = f
for _, d := range nvyx.decoders[DecodeTypeURI] {
if cc, ok := d.(canCheckFile); ok {
if !cc.CanFile(f) {
continue
}
}
aux, err = d.Decode(ctx, p)
if err != nil {
return
}
nn = append(nn, aux...)
nodes = append(nodes, aux...)
_, err = f.Seek(0, 0)
if err != nil {
@@ -105,5 +119,26 @@ func (nvyx *Service) decodeFile(ctx context.Context, p DecodeParams, path string
}
}
providerIdent := strings.Replace(f.Name(), basePath, "", -1)
providerIdent = strings.TrimLeft(providerIdent, "/")
if csv.CanDecodeFile(f) || csv.CanDecodeExt(f.Name()) {
f.Seek(0, 0)
aux, err := csv.Decoder(f, providerIdent)
if err != nil {
return nil, nil, err
}
providers = append(providers, aux)
}
if json.CanDecodeFile(f) || json.CanDecodeExt(f.Name()) {
f.Seek(0, 0)
aux, err := json.Decoder(f, providerIdent)
if err != nil {
return nil, nil, err
}
providers = append(providers, aux)
}
return
}
server/pkg/envoyx/json/decoder.go
+9 -3
View File
@@ -1,6 +1,7 @@
package csv
package json
import (
"context"
"encoding/json"
"io"
"io/ioutil"
@@ -94,6 +95,11 @@ func (d *decoder) Cleanup() error {
return os.Remove(d.src.Name())
}
// SetIdent overwrites the system defined identifier
func (d *decoder) SetIdent(ident string) {
d.ident = ident
}
// Ident returns the assigned identifier
func (d *decoder) Ident() string {
return d.ident
@@ -105,13 +111,13 @@ func (d *decoder) Fields() []string {
}
// Reset resets the decoder to the start
func (d *decoder) Reset() error {
func (d *decoder) Reset(_ context.Context) error {
_, err := d.src.Seek(0, 0)
return err
}
// Next returns the field: value mapping for the next row
func (d *decoder) Next(out map[string]string) (more bool, err error) {
func (d *decoder) Next(_ context.Context, out map[string]string) (more bool, err error) {
err = d.reader.Decode(&out)
if err == io.EOF {
return false, nil
server/pkg/envoyx/json/decoder_test.go
+5 -5
View File
@@ -1,4 +1,4 @@
package csv
package json
import (
"io"
@@ -35,28 +35,28 @@ func TestDecoder(t *testing.T) {
aux := make(map[string]string)
var more bool
more, err = dc.Next(aux)
more, err = dc.Next(nil, aux)
req.NoError(err)
req.True(more)
req.Equal("r1f1", aux["f1"])
req.Equal("r1f2", aux["f2"])
req.Equal("r1f3", aux["f3"])
more, err = dc.Next(aux)
more, err = dc.Next(nil, aux)
req.NoError(err)
req.True(more)
req.Equal("r2f1", aux["f1"])
req.Equal("r2f2", aux["f2"])
req.Equal("r2f3", aux["f3"])
more, err = dc.Next(aux)
more, err = dc.Next(nil, aux)
req.NoError(err)
req.True(more)
req.Equal("r3f1", aux["f1"])
req.Equal("r3f2", aux["f2"])
req.Equal("r3f3", aux["f3"])
more, err = dc.Next(aux)
more, err = dc.Next(nil, aux)
req.NoError(err)
req.False(more)
})
server/pkg/envoyx/utils.gen.go
Generated
+1 -2
View File
@@ -28,7 +28,6 @@ var (
// superNeedyResources is the second level of filtering in case the first
// pass removes everything
superNeedyResources = map[string]bool{
"corteza::compose:module-field": true,
"corteza::compose:record-datasource": true,
"corteza::compose:module-field": true,
}
)
server/tests/envoy/import_export_test.go
+6 -6
View File
@@ -44,7 +44,7 @@ func TestImportExport(t *testing.T) {
t.Run("initial import", func(t *testing.T) {
t.Run("parse configs", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": "file://testdata/full",
@@ -60,7 +60,7 @@ func TestImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
@@ -87,7 +87,7 @@ func TestImportExport(t *testing.T) {
t.Run("export", func(t *testing.T) {
t.Run("export from DB", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
@@ -122,7 +122,7 @@ func TestImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
@@ -141,7 +141,7 @@ func TestImportExport(t *testing.T) {
t.Run("second import", func(t *testing.T) {
t.Run("yaml parse", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": fmt.Sprintf("file://%s", auxFile.Name()),
@@ -157,7 +157,7 @@ func TestImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
server/tests/envoy/main_test.go
+19
View File
@@ -130,12 +130,28 @@ func cleanup(t *testing.T) {
store.TruncateAutomationWorkflows(ctx, defaultStore),
store.TruncateAutomationTriggers(ctx, defaultStore),
truncateRecords(ctx),
)
if err != nil {
t.Fatalf("failed to decode scenario data: %v", err)
}
}
func truncateRecords(ctx context.Context) error {
models, err := defaultDal.SearchModels(ctx)
if err != nil {
return err
}
for _, model := range models {
err = defaultDal.Truncate(ctx, model.ToFilter(), nil)
if err != nil {
return err
}
}
return nil
}
func collect(ee ...error) error {
for _, e := range ee {
if e != nil {
@@ -168,4 +184,7 @@ func initSvc(ctx context.Context) {
systemEnvoy.StoreEncoder{},
automationEnvoy.StoreEncoder{},
)
defaultEnvoy.AddEncoder(envoyx.EncodeTypeIo,
composeEnvoy.CsvEncoder{},
)
}
server/tests/envoy/rbac_rules_test.go
+6 -6
View File
@@ -43,7 +43,7 @@ func TestRbacImportExport(t *testing.T) {
t.Run("initial import", func(t *testing.T) {
t.Run("parse configs", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": "file://testdata/rbac",
@@ -59,7 +59,7 @@ func TestRbacImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
@@ -87,7 +87,7 @@ func TestRbacImportExport(t *testing.T) {
var rules envoyx.NodeSet
t.Run("export", func(t *testing.T) {
t.Run("export from DB", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
@@ -122,7 +122,7 @@ func TestRbacImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
@@ -141,7 +141,7 @@ func TestRbacImportExport(t *testing.T) {
t.Run("second import", func(t *testing.T) {
t.Run("yaml parse", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": fmt.Sprintf("file://%s", auxFile.Name()),
@@ -157,7 +157,7 @@ func TestRbacImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
server/tests/envoy/records_test.go
+232
View File
@@ -0,0 +1,232 @@
package envoy
import (
"context"
"fmt"
"os"
"testing"
"github.com/cortezaproject/corteza/server/compose/dalutils"
"github.com/cortezaproject/corteza/server/compose/envoy"
"github.com/cortezaproject/corteza/server/compose/types"
"github.com/cortezaproject/corteza/server/pkg/dal"
"github.com/cortezaproject/corteza/server/pkg/envoyx"
"github.com/cortezaproject/corteza/server/store"
"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/require"
)
func TestRecordsImportExport(t *testing.T) {
var (
ctx = context.Background()
req = require.New(t)
nodes envoyx.NodeSet
providers []envoyx.Provider
gg *envoyx.DepGraph
err error
)
_ = gg
cleanup(t)
// The test
//
// * imports some YAML files
// * checks the DB state
// * exports the DB into a YAML
// * clears the DB
// * imports the YAML
// * checks the DB state the same way as before
//
// The above outlined flow allows us to trivially check if the data is both
// imported and exported correctly.
//
// The initial step could also manually populate the DB but the YAML import
// is more convenient.
t.Run("initial import", func(t *testing.T) {
t.Run("parse configs", func(t *testing.T) {
nodes, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": "file://testdata/datasource_records",
},
})
req.NoError(err)
})
t.Run("bake", func(t *testing.T) {
gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{
Type: envoyx.EncodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
"dal": defaultDal,
},
}, providers, nodes...)
req.NoError(err)
})
t.Run("import into DB", func(t *testing.T) {
err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{
Type: envoyx.EncodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
"dal": defaultDal,
},
}, gg)
req.NoError(err)
})
assertRecordState(ctx, t, defaultStore, defaultDal, req)
})
// Prepare a temp file where we'll dump the YAML into
auxFile, err := os.CreateTemp(os.TempDir(), "*.csv")
req.NoError(err)
spew.Dump(auxFile.Name())
// defer os.Remove(auxFile.Name())
defer auxFile.Close()
t.Run("export", func(t *testing.T) {
t.Run("export from DB", func(t *testing.T) {
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
"dal": defaultDal,
},
Filter: map[string]envoyx.ResourceFilter{
envoy.ComposeRecordDatasourceAuxType: {
Refs: map[string]envoyx.Ref{
"NamespaceID": {
ResourceType: types.NamespaceResourceType,
Identifiers: envoyx.MakeIdentifiers("test_ns_1"),
Scope: envoyx.Scope{
ResourceType: types.NamespaceResourceType,
Identifiers: envoyx.MakeIdentifiers("test_ns_1")},
},
"ModuleID": {
ResourceType: types.ModuleResourceType,
Identifiers: envoyx.MakeIdentifiers("test_ns_1_mod_1"),
Scope: envoyx.Scope{
ResourceType: types.NamespaceResourceType,
Identifiers: envoyx.MakeIdentifiers("test_ns_1"),
},
},
},
Scope: envoyx.Scope{
ResourceType: types.NamespaceResourceType,
Identifiers: envoyx.MakeIdentifiers("test_ns_1"),
},
},
},
})
req.NoError(err)
})
t.Run("bake", func(t *testing.T) {
gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{
Type: envoyx.EncodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
"dal": defaultDal,
},
}, nil, nodes...)
req.NoError(err)
})
t.Run("write file", func(t *testing.T) {
err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{
Type: envoyx.EncodeTypeIo,
Params: map[string]any{
"writer": auxFile,
},
}, gg)
req.NoError(err)
})
})
cleanup(t)
t.Run("second import", func(t *testing.T) {
t.Run("yaml parse", func(t *testing.T) {
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": "file://testdata/datasource_records",
},
})
req.NoError(err)
_, providers, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": fmt.Sprintf("file://%s", auxFile.Name()),
},
})
req.NoError(err)
for _, p := range providers {
p.SetIdent("records.csv")
}
})
t.Run("bake", func(t *testing.T) {
gg, err = defaultEnvoy.Bake(ctx, envoyx.EncodeParams{
Type: envoyx.EncodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
"dal": defaultDal,
},
}, providers, nodes...)
req.NoError(err)
})
t.Run("run import", func(t *testing.T) {
err = defaultEnvoy.Encode(ctx, envoyx.EncodeParams{
Type: envoyx.EncodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
"dal": defaultDal,
},
}, gg)
req.NoError(err)
})
assertRecordState(ctx, t, defaultStore, defaultDal, req)
})
}
func assertRecordState(ctx context.Context, t *testing.T, s store.Storer, dl dal.FullService, req *require.Assertions) {
t.Run("check state", func(t *testing.T) {
ns, err := store.LookupComposeNamespaceBySlug(ctx, defaultStore, "test_ns_1")
req.NoError(err)
mod, err := store.LookupComposeModuleByNamespaceIDHandle(ctx, defaultStore, ns.ID, "test_ns_1_mod_1")
req.NoError(err)
rr, _, err := dalutils.ComposeRecordsList(ctx, dl, mod, types.RecordFilter{})
req.NoError(err)
req.Len(rr, 8)
compareValues(req, rr[0].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_1_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_1_src_c2"}})
compareValues(req, rr[1].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_2_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_2_src_c2"}})
compareValues(req, rr[2].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_3_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_3_src_c2"}})
compareValues(req, rr[3].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_4_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_4_src_c2"}})
compareValues(req, rr[4].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_5_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_5_src_c2"}})
compareValues(req, rr[5].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_6_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_6_src_c2"}})
compareValues(req, rr[6].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_7_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_7_src_c2"}})
compareValues(req, rr[7].Values, types.RecordValueSet{{Name: "test_ns_1_mod_1_f1", Value: "row_8_src_c1"}, {Name: "test_ns_1_mod_1_f2", Value: "row_8_src_c2"}})
})
}
func compareValues(req *require.Assertions, a, b types.RecordValueSet) {
for _, va := range a {
vb := b.Get(va.Name, va.Place)
req.NotNil(vb)
req.Equal(va.Value, vb.Value)
}
}
server/tests/envoy/res_tr_test.go
+6 -6
View File
@@ -43,7 +43,7 @@ func TestResTrImportExport(t *testing.T) {
t.Run("initial import", func(t *testing.T) {
t.Run("parse configs", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": "file://testdata/locale",
@@ -59,7 +59,7 @@ func TestResTrImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
@@ -88,7 +88,7 @@ func TestResTrImportExport(t *testing.T) {
var translations envoyx.NodeSet
t.Run("export", func(t *testing.T) {
t.Run("export from DB", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeStore,
Params: map[string]any{
"storer": defaultStore,
@@ -122,7 +122,7 @@ func TestResTrImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
@@ -141,7 +141,7 @@ func TestResTrImportExport(t *testing.T) {
t.Run("second import", func(t *testing.T) {
t.Run("yaml parse", func(t *testing.T) {
nodes, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
nodes, _, err = defaultEnvoy.Decode(ctx, envoyx.DecodeParams{
Type: envoyx.DecodeTypeURI,
Params: map[string]any{
"uri": fmt.Sprintf("file://%s", auxFile.Name()),
@@ -157,7 +157,7 @@ func TestResTrImportExport(t *testing.T) {
"storer": defaultStore,
"dal": defaultDal,
},
}, nodes...)
}, nil, nodes...)
req.NoError(err)
})
server/tests/envoy/testdata/datasource_records/def.yaml
Vendored
+22
View File
@@ -0,0 +1,22 @@
namespace:
test_ns_1:
name: Test Namespace 1
modules:
test_ns_1_mod_1:
name: Test Namespace 1 Module 1
fields:
test_ns_1_mod_1_f1:
title: Test Namespace 1 Module 1 Field 1
test_ns_1_mod_1_f2:
title: Test Namespace 1 Module 1 Field 2
kind: Number
source:
- from: records.csv
key: ID
map:
- column: ID
skip: true
- column: test_ns_1_mod_1_f1
field: test_ns_1_mod_1_f1
- column: test_ns_1_mod_1_f2
field: test_ns_1_mod_1_f2
server/tests/envoy/testdata/datasource_records/records.csv
Vendored
+9
View File
@@ -0,0 +1,9 @@
ID,test_ns_1_mod_1_f1,test_ns_1_mod_1_f2
row_1,row_1_src_c1,row_1_src_c2
row_2,row_2_src_c1,row_2_src_c2
row_3,row_3_src_c1,row_3_src_c2
row_4,row_4_src_c1,row_4_src_c2
row_5,row_5_src_c1,row_5_src_c2
row_6,row_6_src_c1,row_6_src_c2
row_7,row_7_src_c1,row_7_src_c2
row_8,row_8_src_c1,row_8_src_c2
1 ID test_ns_1_mod_1_f1 test_ns_1_mod_1_f2
2 row_1 row_1_src_c1 row_1_src_c2
3 row_2 row_2_src_c1 row_2_src_c2
4 row_3 row_3_src_c1 row_3_src_c2
5 row_4 row_4_src_c1 row_4_src_c2
6 row_5 row_5_src_c1 row_5_src_c2
7 row_6 row_6_src_c1 row_6_src_c2
8 row_7 row_7_src_c1 row_7_src_c2
9 row_8 row_8_src_c1 row_8_src_c2