From cfb0822858f8a45b4d53beafda7062b988f7b4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Mon, 8 Mar 2021 10:40:06 +0100 Subject: [PATCH] Fix record import field mapping & validation * Allow undefined record ref validator fn * Remove field mapping fallback if mapping is provided --- compose/rest/record.go | 1 + compose/service/values/validator.go | 10 ++++--- pkg/envoy/resource/compose_record_shaper.go | 28 ++++++++++++++----- pkg/envoy/resource/compose_record_template.go | 8 ++++-- pkg/envoy/yaml/compose_module.go | 5 ++-- pkg/envoy/yaml/compose_module_unmarshal.go | 5 +++- 6 files changed, 40 insertions(+), 17 deletions(-) diff --git a/compose/rest/record.go b/compose/rest/record.go index 2e123a4db..925dd2d44 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -333,6 +333,7 @@ func (ctrl *Record) ImportRun(ctx context.Context, r *request.RecordImportRun) ( strconv.FormatUint(ses.ModuleID, 10), strconv.FormatUint(ses.NamespaceID, 10), ses.Name, + false, resource.MapToMappingTplSet(ses.Fields), ) diff --git a/compose/service/values/validator.go b/compose/service/values/validator.go index 14bee6f4f..48a0dcdfa 100644 --- a/compose/service/values/validator.go +++ b/compose/service/values/validator.go @@ -3,15 +3,16 @@ package values import ( "context" "fmt" - "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/expr" - "github.com/cortezaproject/corteza-server/pkg/slice" - "github.com/cortezaproject/corteza-server/store" "math/big" "net/mail" "net/url" "strings" "time" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/expr" + "github.com/cortezaproject/corteza-server/pkg/slice" + "github.com/cortezaproject/corteza-server/store" ) // Validator package provides tooling to validate @@ -333,6 +334,7 @@ func (vldtr validator) vRecord(ctx context.Context, s store.Storer, v *types.Rec if vldtr.recordRefCheckerFn == nil { return nil } + if ok, err := vldtr.recordRefCheckerFn(ctx, s, v, f, m); err != nil { return e2s(makeInternalErr(f, err)) } else if !ok { diff --git a/pkg/envoy/resource/compose_record_shaper.go b/pkg/envoy/resource/compose_record_shaper.go index 087fad3c8..df869edd0 100644 --- a/pkg/envoy/resource/compose_record_shaper.go +++ b/pkg/envoy/resource/compose_record_shaper.go @@ -52,7 +52,7 @@ func (crt *composeRecordShaper) toResource(def *ComposeRecordTemplate, dt *Resou // Get the bits in order rRaw := &ComposeRecordRaw{ ID: crt.getKey(mr, def.Key), - Values: crt.mapValues(mr, def.FieldMap), + Values: crt.mapValues(mr, def.FieldMap, def.Defaultable), } crt.getTimestamps(rRaw) @@ -70,7 +70,7 @@ func (crt *composeRecordShaper) toResource(def *ComposeRecordTemplate, dt *Resou } // mapValues maps original values based on the provided mapping -func (crt *composeRecordShaper) mapValues(ov map[string]string, fm MappingTplSet) map[string]string { +func (crt *composeRecordShaper) mapValues(ov map[string]string, fm MappingTplSet, defaultable bool) map[string]string { nv := make(map[string]string) // Mappings are provided as a slice since we'll want to do some additional conditional mappings. @@ -80,13 +80,27 @@ func (crt *composeRecordShaper) mapValues(ov map[string]string, fm MappingTplSet mx[m.Cell] = m } - for k, v := range ov { - if m, has := mx[k]; has { - if !m.IsIgnored() { - nv[m.Field] = v + if defaultable { + for k, v := range ov { + if m, has := mx[k]; has { + if !m.IsIgnored() { + nv[m.Field] = v + } + } else { + nv[k] = v + } + } + } else { + if len(mx) > 0 { + for k, v := range ov { + if m, has := mx[k]; has && !m.IsIgnored() { + nv[m.Field] = v + } } } else { - nv[k] = v + for k, v := range ov { + nv[k] = v + } } } diff --git a/pkg/envoy/resource/compose_record_template.go b/pkg/envoy/resource/compose_record_template.go index 8289ce481..9cdfe2afe 100644 --- a/pkg/envoy/resource/compose_record_template.go +++ b/pkg/envoy/resource/compose_record_template.go @@ -27,19 +27,21 @@ type ( // Name is the source name; topically file name Name string // Key determines what defines an identifier - Key []string - FieldMap MappingTplSet + Key []string + FieldMap MappingTplSet + Defaultable bool } ) // NewComposeRecordTemplate returns a record template based on the provided parameters. // // The template is only valid up until the resource shaping; it is not allowed after the fact. -func NewComposeRecordTemplate(modRef, nsRef, name string, fieldMap MappingTplSet, key ...string) *ComposeRecordTemplate { +func NewComposeRecordTemplate(modRef, nsRef, name string, defaultable bool, fieldMap MappingTplSet, key ...string) *ComposeRecordTemplate { r := &ComposeRecordTemplate{base: &base{}} r.Name = name r.Key = key r.FieldMap = fieldMap + r.Defaultable = defaultable r.NsRef = r.AddRef(COMPOSE_NAMESPACE_RESOURCE_TYPE, nsRef) r.ModRef = r.AddRef(COMPOSE_MODULE_RESOURCE_TYPE, modRef).Constraint(r.NsRef) diff --git a/pkg/envoy/yaml/compose_module.go b/pkg/envoy/yaml/compose_module.go index 0b2a14ca6..44536ab92 100644 --- a/pkg/envoy/yaml/compose_module.go +++ b/pkg/envoy/yaml/compose_module.go @@ -14,8 +14,9 @@ type ( composeRecordTpl struct { Source string `yaml:"from"` - Key []string - Mapping mappingTplSet + Key []string + Mapping mappingTplSet + Defaultable bool } composeModule struct { diff --git a/pkg/envoy/yaml/compose_module_unmarshal.go b/pkg/envoy/yaml/compose_module_unmarshal.go index d7dbf3c24..d25ea97dd 100644 --- a/pkg/envoy/yaml/compose_module_unmarshal.go +++ b/pkg/envoy/yaml/compose_module_unmarshal.go @@ -134,7 +134,7 @@ func (wrap composeModule) MarshalEnvoy() ([]resource.Interface, error) { for _, m := range s.Mapping { mtt = append(mtt, &m.MappingTpl) } - crs = resource.NewComposeRecordTemplate(rs.Identifiers().First(), wrap.refNamespace, s.Source, mtt, s.Key...) + crs = resource.NewComposeRecordTemplate(rs.Identifiers().First(), wrap.refNamespace, s.Source, s.Defaultable, mtt, s.Key...) } return envoy.CollectNodes( @@ -161,6 +161,9 @@ func (rt *composeRecordTpl) UnmarshalYAML(n *yaml.Node) error { }) } + case "defaultable": + return y7s.DecodeScalar(v, "defaultable field mapping", &rt.Defaultable) + case "mapping", "map": rt.Mapping = make(mappingTplSet, 0, 20) // When provided as a sequence node, map the fields based on the index.