From d70916705d2d3cbbd754970cc8be4dd16778cdfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 6 Apr 2021 22:59:08 +0200 Subject: [PATCH] Post testing Envoy tweaks * ComposeRecord encoding with reference to self, * improved ComposeRecord reference handling, * tweaked Select ComposeModuleField validation. --- compose/rest/record.go | 1 + compose/service/import_session.go | 11 ++ compose/service/record.go | 1 + compose/service/values/validator.go | 7 +- pkg/envoy/csv/decoder.go | 25 ++- pkg/envoy/json/decoder.go | 20 ++- pkg/envoy/resource/compose_record.go | 6 + pkg/envoy/resource/compose_record_shaper.go | 4 + pkg/envoy/resource/dataset.go | 1 + pkg/envoy/store/compose_module_marshal.go | 19 +- pkg/envoy/store/compose_record_marshal.go | 63 +++++-- tests/envoy/data_shaping_test.go | 170 ++++++++++++++++++ tests/envoy/main.go | 13 ++ .../csv_fieldtypes/1100_modules.yaml | 65 +++++++ .../data_shaping/csv_fieldtypes/mod1.csv | 3 + .../data_shaping/csv_refs/1100_modules.yaml | 26 +++ .../testdata/data_shaping/csv_refs/mod1.csv | 5 + .../csv_selfref/1100_modules.yaml | 31 ++++ .../data_shaping/csv_selfref/mod1.csv | 3 + .../jsonl_fieldtypes/1100_modules.yaml | 65 +++++++ .../data_shaping/jsonl_fieldtypes/mod1.jsonl | 2 + .../jsonl_selfref/1100_modules.yaml | 31 ++++ .../data_shaping/jsonl_selfref/mod1.jsonl | 2 + 23 files changed, 544 insertions(+), 30 deletions(-) create mode 100644 tests/envoy/testdata/data_shaping/csv_fieldtypes/1100_modules.yaml create mode 100644 tests/envoy/testdata/data_shaping/csv_fieldtypes/mod1.csv create mode 100644 tests/envoy/testdata/data_shaping/csv_refs/1100_modules.yaml create mode 100644 tests/envoy/testdata/data_shaping/csv_refs/mod1.csv create mode 100644 tests/envoy/testdata/data_shaping/csv_selfref/1100_modules.yaml create mode 100644 tests/envoy/testdata/data_shaping/csv_selfref/mod1.csv create mode 100644 tests/envoy/testdata/data_shaping/jsonl_fieldtypes/1100_modules.yaml create mode 100644 tests/envoy/testdata/data_shaping/jsonl_fieldtypes/mod1.jsonl create mode 100644 tests/envoy/testdata/data_shaping/jsonl_selfref/1100_modules.yaml create mode 100644 tests/envoy/testdata/data_shaping/jsonl_selfref/mod1.jsonl diff --git a/compose/rest/record.go b/compose/rest/record.go index 5a3bbf1cd..b4cb4ae27 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -335,6 +335,7 @@ func (ctrl *Record) ImportRun(ctx context.Context, r *request.RecordImportRun) ( ses.Name, false, resource.MapToMappingTplSet(ses.Fields), + ses.Key, ) // Shape the data diff --git a/compose/service/import_session.go b/compose/service/import_session.go index 10c8238b5..30c72adef 100644 --- a/compose/service/import_session.go +++ b/compose/service/import_session.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "strings" "sync" "time" @@ -102,9 +103,19 @@ func (svc *importSession) Create(ctx context.Context, f io.ReadSeeker, name, con return nil, fmt.Errorf("compose.service.RecordImportFormatNotSupported") } + prepKey := func(k string) string { + return strings.TrimSpace(strings.ToLower(k)) + } + sh.Progress.EntryCount = n.P.Count() for _, f := range n.P.Fields() { sh.Fields[f] = "" + + // @todo improve this bit + k := prepKey(f) + if k == "id" || k == "recordid" { + sh.Key = f + } } // Create it diff --git a/compose/service/record.go b/compose/service/record.go index 5ee7da3df..9779da1e9 100644 --- a/compose/service/record.go +++ b/compose/service/record.go @@ -109,6 +109,7 @@ type ( OnError string `json:"onError"` Fields map[string]string `json:"fields"` + Key string `json:"key"` Progress *RecordImportProgress `json:"progress"` CreatedAt time.Time `json:"createdAt"` diff --git a/compose/service/values/validator.go b/compose/service/values/validator.go index 48a0dcdfa..e56432307 100644 --- a/compose/service/values/validator.go +++ b/compose/service/values/validator.go @@ -372,7 +372,12 @@ func (vldtr validator) vSelect(v *types.RecordValue, f *types.ModuleField, r *ty sbm[value] = true } } - + case types.ModuleFieldOptions: + if value, has := c["value"]; has { + if value, ok := value.(string); ok { + sbm[value] = true + } + } } } } diff --git a/pkg/envoy/csv/decoder.go b/pkg/envoy/csv/decoder.go index b2277f1ce..763ed0b90 100644 --- a/pkg/envoy/csv/decoder.go +++ b/pkg/envoy/csv/decoder.go @@ -21,6 +21,8 @@ type ( c *csv.Reader header []string count uint64 + + buff bytes.Buffer } ) @@ -54,13 +56,12 @@ func (c *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts var buff bytes.Buffer // So we can reset to the start of the reader - tr := io.TeeReader(r, &buff) - err := cr.prepare(tr) + err := cr.prepare(io.TeeReader(r, &buff)) if err != nil { return nil, err } - cr.c = csv.NewReader(&buff) + cr.c = csv.NewReader(io.TeeReader(&buff, &cr.buff)) // The first one is a header, so let's just get rid of it _, err = cr.c.Read() @@ -100,6 +101,24 @@ func (cr *reader) Fields() []string { return cr.header } +func (cr *reader) Reset() error { + if len(cr.buff.Bytes()) == 0 { + return nil + } + + buff := cr.buff + cr.buff = bytes.Buffer{} + + cr.c = csv.NewReader(io.TeeReader(&buff, &cr.buff)) + // The first one is a header, so let's just get rid of it + _, err := cr.c.Read() + if err != nil { + return err + } + + return nil +} + // Next returns the field: value mapping for the next row func (cr *reader) Next() (map[string]string, error) { mr := make(map[string]string) diff --git a/pkg/envoy/json/decoder.go b/pkg/envoy/json/decoder.go index c30050837..373607c07 100644 --- a/pkg/envoy/json/decoder.go +++ b/pkg/envoy/json/decoder.go @@ -21,6 +21,8 @@ type ( d *json.Decoder header []string count uint64 + + buff bytes.Buffer } ) @@ -58,13 +60,12 @@ func (d *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts var buff bytes.Buffer // So we can reset to the start of the reader - tr := io.TeeReader(r, &buff) - err := jr.prepare(tr) + err := jr.prepare(io.TeeReader(r, &buff)) if err != nil { return nil, err } - jr.d = json.NewDecoder(&buff) + jr.d = json.NewDecoder(io.TeeReader(&buff, &jr.buff)) return []resource.Interface{resource.NewResourceDataset(do.Name, jr)}, nil } @@ -102,6 +103,19 @@ func (jr *reader) prepare(r io.Reader) (err error) { return nil } +func (jr *reader) Reset() error { + if len(jr.buff.Bytes()) == 0 { + return nil + } + + buff := jr.buff + jr.buff = bytes.Buffer{} + + jr.d = json.NewDecoder(io.TeeReader(&buff, &jr.buff)) + + return nil +} + // Fields returns every available field in this dataset func (jr *reader) Fields() []string { return jr.header diff --git a/pkg/envoy/resource/compose_record.go b/pkg/envoy/resource/compose_record.go index 8e16e4711..bcaf9b74a 100644 --- a/pkg/envoy/resource/compose_record.go +++ b/pkg/envoy/resource/compose_record.go @@ -1,6 +1,8 @@ package resource import ( + "fmt" + "github.com/cortezaproject/corteza-server/compose/types" ) @@ -69,3 +71,7 @@ func (r *ComposeRecord) SetUserFlakes(uu UserstampIndex) { // Set user refs as wildflag, indicating it refers to any user resource r.AddRef(USER_RESOURCE_TYPE, "*") } + +func ComposeRecordErrUnresolved(ii Identifiers) error { + return fmt.Errorf("compose record unresolved %v", ii.StringSlice()) +} diff --git a/pkg/envoy/resource/compose_record_shaper.go b/pkg/envoy/resource/compose_record_shaper.go index df869edd0..6ee3d3351 100644 --- a/pkg/envoy/resource/compose_record_shaper.go +++ b/pkg/envoy/resource/compose_record_shaper.go @@ -40,6 +40,10 @@ func (crt *composeRecordShaper) Shape(rr []Interface) ([]Interface, error) { func (crt *composeRecordShaper) toResource(def *ComposeRecordTemplate, dt *ResourceDataset) Interface { w := func(f func(r *ComposeRecordRaw) error) error { + if err := dt.P.Reset(); err != nil { + return err + } + for { mr, err := dt.P.Next() if err != nil { diff --git a/pkg/envoy/resource/dataset.go b/pkg/envoy/resource/dataset.go index b141c03dd..eb2448269 100644 --- a/pkg/envoy/resource/dataset.go +++ b/pkg/envoy/resource/dataset.go @@ -4,6 +4,7 @@ type ( provider interface { Fields() []string Count() uint64 + Reset() error Next() (map[string]string, error) } diff --git a/pkg/envoy/store/compose_module_marshal.go b/pkg/envoy/store/compose_module_marshal.go index 4004085aa..3c2f64c5a 100644 --- a/pkg/envoy/store/compose_module_marshal.go +++ b/pkg/envoy/store/compose_module_marshal.go @@ -92,6 +92,15 @@ func (n *composeModule) Encode(ctx context.Context, pl *payload) (err error) { res.ID = NextID() } + res.NamespaceID = n.relNS.ID + if res.NamespaceID <= 0 { + ns := resource.FindComposeNamespace(pl.state.ParentResources, n.res.RefNs.Identifiers) + res.NamespaceID = ns.ID + } + if res.NamespaceID <= 0 { + return resource.ComposeNamespaceErrUnresolved(n.res.RefNs.Identifiers) + } + if pl.state.Conflicting { return nil } @@ -111,16 +120,6 @@ func (n *composeModule) Encode(ctx context.Context, pl *payload) (err error) { } } - // Namespace - res.NamespaceID = n.relNS.ID - if res.NamespaceID <= 0 { - ns := resource.FindComposeNamespace(pl.state.ParentResources, n.res.RefNs.Identifiers) - res.NamespaceID = ns.ID - } - if res.NamespaceID <= 0 { - return resource.ComposeNamespaceErrUnresolved(n.res.RefNs.Identifiers) - } - // Fields var originalFields types.ModuleFieldSet if n.mod != nil && n.mod.Fields != nil { diff --git a/pkg/envoy/store/compose_record_marshal.go b/pkg/envoy/store/compose_record_marshal.go index 647cf679b..c43925c79 100644 --- a/pkg/envoy/store/compose_record_marshal.go +++ b/pkg/envoy/store/compose_record_marshal.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "fmt" "strconv" "time" @@ -157,9 +158,15 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) { u := ur.U ux[strconv.FormatUint(u.ID, 10)] = u.ID - ux[u.Handle] = u.ID - ux[u.Name] = u.ID - ux[u.Email] = u.ID + if u.Handle != "" { + ux[u.Handle] = u.ID + } + if u.Name != "" { + ux[u.Name] = u.ID + } + if u.Email != "" { + ux[u.Email] = u.ID + } } // Next all of the encoded users. // If identifiers overwrite eachother, that's fine. @@ -179,7 +186,18 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) { createAcChecked := false updateAcChecked := false + getKey := func(i int, k string) string { + if k == "" { + return strconv.FormatInt(int64(i), 10) + } + + return k + } + + i := -1 return n.res.Walker(func(r *resource.ComposeRecordRaw) error { + i++ + // So we don't have to worry about nil cfg := r.Config if cfg == nil { @@ -223,7 +241,7 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) { } rec := &types.Record{ - ID: im[r.ID], + ID: im[getKey(i, r.ID)], NamespaceID: nsID, ModuleID: mod.ID, CreatedAt: time.Now(), @@ -236,13 +254,14 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) { exists = old != nil } - if rec.ID <= 0 && exists { + if rec.ID == 0 && exists { rec.ID = rm[r.ID].ID - } else { + } + if rec.ID == 0 { rec.ID = NextID() } - im[r.ID] = rec.ID + im[getKey(i, r.ID)] = rec.ID if pl.state.Conflicting { return nil @@ -289,13 +308,31 @@ func (n *composeRecord) Encode(ctx context.Context, pl *payload) (err error) { } f := mod.Fields.FindByName(k) - if f != nil && f.Kind == "User" { - uID := ux[v] - if uID == 0 { - return resource.UserErrUnresolved(resource.MakeIdentifiers(v)) + if f != nil { + switch f.Kind { + case "User": + uID := ux[v] + if uID == 0 { + return resource.UserErrUnresolved(resource.MakeIdentifiers(v)) + } + rv.Value = strconv.FormatUint(uID, 10) + rv.Ref = uID + + case "Record": + // if self... + if n.res.RefMod.Identifiers.HasAny(resource.MakeIdentifiers(f.Options.String("module"))) { + rID := im[v] + if rID == 0 { + return resource.ComposeRecordErrUnresolved(resource.MakeIdentifiers(v)) + } + rv.Value = strconv.FormatUint(rID, 10) + rv.Ref = rID + } else { + // not self... + // @todo... + return errors.New("record cross referencing not yet supported") + } } - rv.Value = strconv.FormatUint(uID, 10) - rv.Ref = uID } rvs = append(rvs, rv) diff --git a/tests/envoy/data_shaping_test.go b/tests/envoy/data_shaping_test.go index 7789fec84..d3d1305e5 100644 --- a/tests/envoy/data_shaping_test.go +++ b/tests/envoy/data_shaping_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path" + "strconv" "testing" "github.com/cortezaproject/corteza-server/compose/types" @@ -23,6 +24,9 @@ func TestDataShaping(t *testing.T) { cases = []string{ "csv_simple", "jsonl_simple", + + "csv_selfref", + "jsonl_selfref", } ) @@ -81,3 +85,169 @@ func TestDataShaping(t *testing.T) { }) } } + +func TestDataShaping_fieldTypes(t *testing.T) { + var ( + ctx = auth.SetSuperUserContext(context.Background()) + s = initStore(ctx, t) + err error + + cases = []string{ + "csv_fieldtypes", + "jsonl_fieldtypes", + } + ) + + ni := uint64(10) + su.NextID = func() uint64 { + ni++ + return ni + } + + for _, c := range cases { + t.Run(fmt.Sprintf("record shaping; data_shaping_field types/%s", c), func(t *testing.T) { + var ( + req = require.New(t) + ) + + truncateStore(ctx, s, t) + err = collect( + err, + storeRole(ctx, s, 1, "everyone"), + storeRole(ctx, s, 2, "admins"), + ) + if err != nil { + t.Fatal(err.Error()) + } + + nn, err := decodeDirectory(ctx, path.Join("data_shaping", c)) + req.NoError(err) + + crs := resource.ComposeRecordShaper() + nn, err = resource.Shape(nn, crs) + req.NoError(err) + + req.NoError(encode(ctx, s, nn)) + + ns, err := store.LookupComposeNamespaceBySlug(ctx, s, "ns1") + req.NotNil(ns) + ms, err := loadComposeModuleFull(ctx, s, req, ns.ID, "mod1") + req.NotNil(ms) + + rr, _, err := store.SearchComposeRecords(ctx, s, ms, types.RecordFilter{}) + req.NoError(err) + req.Len(rr, 2) + + r1 := rr[0] + r2 := rr[1] + + req.Len(r1.Values, 7) + req.Equal("1", r1.Values.Get("f_bool", 0).Value) + req.Equal("2021-04-01T10:00:00Z", r1.Values.Get("f_datetime", 0).Value) + req.Equal("mail@test.tld", r1.Values.Get("f_email", 0).Value) + req.Equal("1.23", r1.Values.Get("f_number", 0).Value) + req.Equal("opt_2", r1.Values.Get("f_select", 0).Value) + req.Equal("test here", r1.Values.Get("f_string", 0).Value) + req.Equal("https://test.tld", r1.Values.Get("f_url", 0).Value) + + req.Len(r2.Values, 7) + req.Equal("", r2.Values.Get("f_bool", 0).Value) + req.Equal("2021-04-02T10:00:00Z", r2.Values.Get("f_datetime", 0).Value) + req.Equal("mail@test.tld", r2.Values.Get("f_email", 0).Value) + req.Equal("20", r2.Values.Get("f_number", 0).Value) + req.Equal("opt_3", r2.Values.Get("f_select", 0).Value) + req.Equal("test here", r2.Values.Get("f_string", 0).Value) + req.Equal("https://test.tld", r2.Values.Get("f_url", 0).Value) + + s.TruncateComposeRecords(ctx, ms) + }) + } +} + +func TestDataShaping_refs(t *testing.T) { + var ( + ctx = auth.SetSuperUserContext(context.Background()) + s = initStore(ctx, t) + err error + + cases = []string{ + "csv_refs", + } + ) + + ni := uint64(10) + su.NextID = func() uint64 { + ni++ + return ni + } + + for _, c := range cases { + t.Run(fmt.Sprintf("record shaping; data_shaping_field types/%s", c), func(t *testing.T) { + var ( + req = require.New(t) + ) + + truncateStore(ctx, s, t) + err = collect( + err, + storeRole(ctx, s, 1, "everyone"), + storeRole(ctx, s, 2, "admins"), + + storeUser(ctx, s, 201, "u1"), + storeUser(ctx, s, 202, "u2"), + ) + if err != nil { + t.Fatal(err.Error()) + } + + nn, err := decodeDirectory(ctx, path.Join("data_shaping", c)) + req.NoError(err) + + crs := resource.ComposeRecordShaper() + nn, err = resource.Shape(nn, crs) + req.NoError(err) + + req.NoError(encode(ctx, s, nn)) + + ns, err := store.LookupComposeNamespaceBySlug(ctx, s, "ns1") + req.NotNil(ns) + ms, err := loadComposeModuleFull(ctx, s, req, ns.ID, "mod1") + req.NotNil(ms) + + rr, _, err := store.SearchComposeRecords(ctx, s, ms, types.RecordFilter{}) + req.NoError(err) + req.Len(rr, 4) + + r1 := rr[0] + r2 := rr[1] + r3 := rr[2] + r4 := rr[3] + + req.Len(r1.Values, 2) + req.Equal(strconv.FormatUint(r1.ID, 10), r1.Values.Get("f_record", 0).Value) + req.Equal(r1.ID, r1.Values.Get("f_record", 0).Ref) + req.Equal("201", r1.Values.Get("f_user", 0).Value) + req.Equal(uint64(201), r1.Values.Get("f_user", 0).Ref) + + req.Len(r2.Values, 2) + req.Equal(strconv.FormatUint(r2.ID, 10), r2.Values.Get("f_record", 0).Value) + req.Equal(r2.ID, r2.Values.Get("f_record", 0).Ref) + req.Equal("202", r2.Values.Get("f_user", 0).Value) + req.Equal(uint64(202), r2.Values.Get("f_user", 0).Ref) + + req.Len(r3.Values, 2) + req.Equal(strconv.FormatUint(r4.ID, 10), r3.Values.Get("f_record", 0).Value) + req.Equal(r4.ID, r3.Values.Get("f_record", 0).Ref) + req.Equal("201", r3.Values.Get("f_user", 0).Value) + req.Equal(uint64(201), r3.Values.Get("f_user", 0).Ref) + + req.Len(r4.Values, 2) + req.Equal(strconv.FormatUint(r3.ID, 10), r4.Values.Get("f_record", 0).Value) + req.Equal(r3.ID, r4.Values.Get("f_record", 0).Ref) + req.Equal("202", r4.Values.Get("f_user", 0).Value) + req.Equal(uint64(202), r4.Values.Get("f_user", 0).Ref) + + s.TruncateComposeRecords(ctx, ms) + }) + } +} diff --git a/tests/envoy/main.go b/tests/envoy/main.go index 34891c3dc..74ffae67a 100644 --- a/tests/envoy/main.go +++ b/tests/envoy/main.go @@ -224,6 +224,19 @@ func storeRole(ctx context.Context, s store.Storer, rID uint64, ss ...string) er return store.CreateRole(ctx, s, r) } +func storeUser(ctx context.Context, s store.Storer, rID uint64, ss ...string) error { + u := &stypes.User{ + ID: rID, + } + if len(ss) > 0 { + u.Handle = ss[0] + } + if len(ss) > 1 { + u.Name = ss[1] + } + return store.CreateUser(ctx, s, u) +} + // // // // // // Misc. helpers // collect collects all errors from different call responses diff --git a/tests/envoy/testdata/data_shaping/csv_fieldtypes/1100_modules.yaml b/tests/envoy/testdata/data_shaping/csv_fieldtypes/1100_modules.yaml new file mode 100644 index 000000000..ff78113c8 --- /dev/null +++ b/tests/envoy/testdata/data_shaping/csv_fieldtypes/1100_modules.yaml @@ -0,0 +1,65 @@ +namespaces: + ns1: + name: ns1 name + +modules: + mod1: + fields: + f_bool: + label: f_bool label + kind: Bool + f_datetime: + label: f_datetime label + kind: DateTime + f_email: + label: f_email label + kind: Email + f_number: + label: f_number label + kind: Number + options: + precision: 2 + f_select: + label: f_select label + kind: Select + options: + options: + - text: opt1 label + value: opt_1 + - text: opt2 label + value: opt_2 + - text: opt3 label + value: opt_3 + f_string: + label: f_string label + kind: String + f_url: + label: f_url label + kind: Url + + records: + source: mod1.csv + key: id + mapping: + id: / + c_bool: + field: f_bool + c_datetime: + field: f_datetime + c_email: + field: f_email + c_number: + field: f_number + c_select: + field: f_select + c_string: + field: f_string + c_url: + field: f_url + c1: + field: f1 + c2: + field: f2 + + + diff --git a/tests/envoy/testdata/data_shaping/csv_fieldtypes/mod1.csv b/tests/envoy/testdata/data_shaping/csv_fieldtypes/mod1.csv new file mode 100644 index 000000000..f0f443c8e --- /dev/null +++ b/tests/envoy/testdata/data_shaping/csv_fieldtypes/mod1.csv @@ -0,0 +1,3 @@ +id,c_bool,c_datetime,c_email,c_number,c_select,c_string,c_url +1,true,2021-04-01T10:00:00Z,mail@test.tld,1.23,opt_2,test here,https://test.tld +2,false,2021-04-02T10:00:00Z,mail@test.tld,20,opt_3,test here,https://test.tld diff --git a/tests/envoy/testdata/data_shaping/csv_refs/1100_modules.yaml b/tests/envoy/testdata/data_shaping/csv_refs/1100_modules.yaml new file mode 100644 index 000000000..09ebe6db9 --- /dev/null +++ b/tests/envoy/testdata/data_shaping/csv_refs/1100_modules.yaml @@ -0,0 +1,26 @@ +namespaces: + ns1: + name: ns1 name + +modules: + mod1: + fields: + f_record: + label: f_record label + kind: Record + options: + module: mod1 + + f_user: + label: f_user label + kind: User + + records: + source: mod1.csv + key: id + mapping: + id: / + c_record: + field: f_record + c_user: + field: f_user \ No newline at end of file diff --git a/tests/envoy/testdata/data_shaping/csv_refs/mod1.csv b/tests/envoy/testdata/data_shaping/csv_refs/mod1.csv new file mode 100644 index 000000000..bb5869194 --- /dev/null +++ b/tests/envoy/testdata/data_shaping/csv_refs/mod1.csv @@ -0,0 +1,5 @@ +id,c_record,c_user +101,101,u1 +102,102,u2 +103,104,u1 +104,103,u2 diff --git a/tests/envoy/testdata/data_shaping/csv_selfref/1100_modules.yaml b/tests/envoy/testdata/data_shaping/csv_selfref/1100_modules.yaml new file mode 100644 index 000000000..fd7892a5e --- /dev/null +++ b/tests/envoy/testdata/data_shaping/csv_selfref/1100_modules.yaml @@ -0,0 +1,31 @@ +namespaces: + ns1: + name: ns1 name + +modules: + mod1: + fields: + f1: + label: f1 label + kind: String + f2: + label: f2 label + kind: String + f3: + label: f3 label + kind: Record + options: + labelField: f1 + module: mod1 + queryFields: + - f1 + + records: + source: mod1.csv + key: id + mapping: + id: / + c1: + field: f1 + c2: + field: f2 diff --git a/tests/envoy/testdata/data_shaping/csv_selfref/mod1.csv b/tests/envoy/testdata/data_shaping/csv_selfref/mod1.csv new file mode 100644 index 000000000..33556aaae --- /dev/null +++ b/tests/envoy/testdata/data_shaping/csv_selfref/mod1.csv @@ -0,0 +1,3 @@ +id,c1,c2 +1,c1.v1,c2.v1 +2,c1.v2,c2.v2 diff --git a/tests/envoy/testdata/data_shaping/jsonl_fieldtypes/1100_modules.yaml b/tests/envoy/testdata/data_shaping/jsonl_fieldtypes/1100_modules.yaml new file mode 100644 index 000000000..712380a31 --- /dev/null +++ b/tests/envoy/testdata/data_shaping/jsonl_fieldtypes/1100_modules.yaml @@ -0,0 +1,65 @@ +namespaces: + ns1: + name: ns1 name + +modules: + mod1: + fields: + f_bool: + label: f_bool label + kind: Bool + f_datetime: + label: f_datetime label + kind: DateTime + f_email: + label: f_email label + kind: Email + f_number: + label: f_number label + kind: Number + options: + precision: 2 + f_select: + label: f_select label + kind: Select + options: + options: + - text: opt1 label + value: opt_1 + - text: opt2 label + value: opt_2 + - text: opt3 label + value: opt_3 + f_string: + label: f_string label + kind: String + f_url: + label: f_url label + kind: Url + + records: + source: mod1.jsonl + key: id + mapping: + id: / + c_bool: + field: f_bool + c_datetime: + field: f_datetime + c_email: + field: f_email + c_number: + field: f_number + c_select: + field: f_select + c_string: + field: f_string + c_url: + field: f_url + c1: + field: f1 + c2: + field: f2 + + + diff --git a/tests/envoy/testdata/data_shaping/jsonl_fieldtypes/mod1.jsonl b/tests/envoy/testdata/data_shaping/jsonl_fieldtypes/mod1.jsonl new file mode 100644 index 000000000..b52db3725 --- /dev/null +++ b/tests/envoy/testdata/data_shaping/jsonl_fieldtypes/mod1.jsonl @@ -0,0 +1,2 @@ +{"id": "1", "c_bool": "true", "c_datetime": "2021-04-01T10:00:00Z", "c_email": "mail@test.tld", "c_number": "1.23", "c_select": "opt_2", "c_string": "test here", "c_url": "https://test.tld"} +{"id": "2", "c_bool": "false", "c_datetime": "2021-04-02T10:00:00Z", "c_email": "mail@test.tld", "c_number": "20", "c_select": "opt_3", "c_string": "test here", "c_url": "https://test.tld"} \ No newline at end of file diff --git a/tests/envoy/testdata/data_shaping/jsonl_selfref/1100_modules.yaml b/tests/envoy/testdata/data_shaping/jsonl_selfref/1100_modules.yaml new file mode 100644 index 000000000..5a162cb5a --- /dev/null +++ b/tests/envoy/testdata/data_shaping/jsonl_selfref/1100_modules.yaml @@ -0,0 +1,31 @@ +namespaces: + ns1: + name: ns1 name + +modules: + mod1: + fields: + f1: + label: f1 label + kind: String + f2: + label: f2 label + kind: String + f3: + label: f3 label + kind: Record + options: + labelField: f1 + module: mod1 + queryFields: + - f1 + + records: + source: mod1.jsonl + key: id + mapping: + id: / + c1: + field: f1 + c2: + field: f2 diff --git a/tests/envoy/testdata/data_shaping/jsonl_selfref/mod1.jsonl b/tests/envoy/testdata/data_shaping/jsonl_selfref/mod1.jsonl new file mode 100644 index 000000000..660375b78 --- /dev/null +++ b/tests/envoy/testdata/data_shaping/jsonl_selfref/mod1.jsonl @@ -0,0 +1,2 @@ +{"id": "1", "c1": "c1.v1", "c2": "c2.v1"} +{"id": "2", "c1": "c1.v2", "c2": "c2.v2"} \ No newline at end of file