DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Fixed mapper on structure, data sync.

Browse Source
This commit is contained in:
Peter Grlica
2020-11-09 17:15:25 +01:00
parent 2690427f5d
commit d5c5c52546
5 changed files with 108 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
federation/service/mapper.go
View File

@@ -14,10 +14,22 @@ type (
//
// mostly, there will be less mapped fields on the destination
// side, so start looping from here
func (m *Mapper) Merge(in *ct.RecordValueSet, out *ct.RecordValueSet) {
func (m *Mapper) Merge(in *ct.RecordValueSet, out *ct.RecordValueSet, mappings *types.ModuleFieldMappingSet) {
var match *types.ModuleFieldMapping
for _, destVal := range *out {
// preset the value, since we're working with
// a pointer to an existing structure, could be some leftovers
destVal.Value = ""
// find origin field
if match, _ = mappings.FindByName(destVal.Name, types.ModuleFieldMappingSetFindTypeDestination); match == nil {
continue
}
// use the found origin, so the mapping is correct
for _, origVal := range *in {
if destVal.Name == origVal.Name {
if origVal.Name == match.Origin.Name {
destVal.Value = origVal.Value
break
}

61
federation/service/mapper_test.go Normal file
View File

@@ -0,0 +1,61 @@
package service
import (
"encoding/json"
"testing"
ct "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/federation/types"
"github.com/stretchr/testify/require"
)
func TestMapper_merge(t *testing.T) {
var (
// out = &
tcc = []struct {
name string
m string
in ct.RecordValueSet
out ct.RecordValueSet
expect interface{}
}{
{
"merge_missing_destination_field",
`[{"origin":{"kind":"String","name":"Description","label":"Description","isMulti":false},"destination":{"kind":"String","name":"Name","label":"Description","isMulti":false}},{"origin":{"kind":"Url","name":"Facebook","label":"Facebook","isMulti":false},"destination":{"kind":"Url","name":"Fb","label":"Facebook","isMulti":false}}]`,
ct.RecordValueSet{&ct.RecordValue{Name: "Facebook", Value: "https://fb.com/user_1"}, &ct.RecordValue{Name: "Phone", Value: "000 111 222"}, &ct.RecordValue{Name: "Twitter", Value: "https://twitter.com/@russian_bot_0"}},
ct.RecordValueSet{&ct.RecordValue{Name: "Name", Value: ""}, &ct.RecordValue{Name: "Fb", Value: ""}},
ct.RecordValueSet{&ct.RecordValue{Name: "Name", Value: ""}, &ct.RecordValue{Name: "Fb", Value: "https://fb.com/user_1"}},
},
{
"merge_empty_origin",
`[{"origin":{"kind":"String","name":"Description","label":"Description","isMulti":false},"destination":{"kind":"String","name":"Name","label":"Description","isMulti":false}},{"origin":{"kind":"Url","name":"Facebook","label":"Facebook","isMulti":false},"destination":{"kind":"Url","name":"Fb","label":"Facebook","isMulti":false}}]`,
ct.RecordValueSet{},
ct.RecordValueSet{&ct.RecordValue{Name: "Name", Value: ""}, &ct.RecordValue{Name: "Fb", Value: ""}},
ct.RecordValueSet{&ct.RecordValue{Name: "Name", Value: ""}, &ct.RecordValue{Name: "Fb", Value: ""}},
},
{
"merge_empty_destination",
`[{"origin":{"kind":"String","name":"Description","label":"Description","isMulti":false},"destination":{"kind":"String","name":"Name","label":"Description","isMulti":false}},{"origin":{"kind":"Url","name":"Facebook","label":"Facebook","isMulti":false},"destination":{"kind":"Url","name":"Fb","label":"Facebook","isMulti":false}}]`,
ct.RecordValueSet{&ct.RecordValue{Name: "Facebook", Value: "https://fb.com/user_1"}, &ct.RecordValue{Name: "Phone", Value: "000 111 222"}, &ct.RecordValue{Name: "Twitter", Value: "https://twitter.com/@russian_bot_0"}},
ct.RecordValueSet{},
ct.RecordValueSet{},
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
mapper = &Mapper{}
)
// dont catch any helper errors
mm := &types.ModuleFieldMappingSet{}
json.Unmarshal([]byte(tc.m), mm)
mapper.Merge(&tc.in, &tc.out, mm)
req.Equal(tc.out, tc.expect)
})
}
}

4
federation/service/sync_worker_data.go
View File

@@ -27,6 +27,7 @@ type (
ComposeModuleID uint64
ComposeNamespaceID uint64
NodeBaseURL string
ModuleMappings *types.ModuleFieldMappingSet
ModuleMappingValues *ct.RecordValueSet
SyncService *Sync
}
@@ -117,6 +118,7 @@ func (w *syncWorkerData) PrepareForNodes(ctx context.Context, urls chan Url) {
NodeID: n.SharedNodeID,
ComposeModuleID: mappings.ComposeModuleID,
ComposeNamespaceID: mappings.ComposeNamespaceID,
ModuleMappings: &mappings.FieldMapping,
ModuleMappingValues: &mappingValues,
NodeBaseURL: n.BaseURL,
SyncService: w.syncService,
@@ -255,7 +257,7 @@ func (dp *dataProcesser) Process(ctx context.Context, payload []byte) (int, erro
}
for _, er := range o {
dp.SyncService.mapper.Merge(&er.Values, dp.ModuleMappingValues)
dp.SyncService.mapper.Merge(&er.Values, dp.ModuleMappingValues, dp.ModuleMappings)
rec := &ct.Record{
ModuleID: dp.ComposeModuleID,

8
federation/service/sync_worker_structure.go
View File

@@ -206,11 +206,9 @@ func (w *syncWorkerStructure) Watch(ctx context.Context, delay time.Duration, li
func (dp *structureProcesser) Process(ctx context.Context, payload []byte) (int, error) {
processed := 0
now := time.Now()
o, err := decoder.DecodeFederationModuleSync([]byte(payload))
if err != nil {
spew.Dump("ERR", err)
return processed, err
}
@@ -218,13 +216,13 @@ func (dp *structureProcesser) Process(ctx context.Context, payload []byte) (int,
return processed, nil
}
for i, em := range o {
for _, em := range o {
new := &types.SharedModule{
NodeID: dp.NodeID,
ExternalFederationModuleID: em.ID,
Fields: em.Fields,
Handle: fmt.Sprintf("Handle %d %d", i, now.Unix()),
Name: fmt.Sprintf("Name %d %d", i, now.Unix()),
Handle: em.Handle,
Name: em.Name,
}
existing, err := dp.SyncService.LookupSharedModule(ctx, new)

27
federation/types/module_field_mapping.go
View File

@@ -7,6 +7,11 @@ import (
"fmt"
)
const (
ModuleFieldMappingSetFindTypeOrigin ModuleFieldMappingSetFindType = iota
ModuleFieldMappingSetFindTypeDestination
)
type (
ModuleFieldMappingSet []*ModuleFieldMapping
@@ -14,8 +19,30 @@ type (
Origin ModuleField `json:"origin"`
Destination ModuleField `json:"destination"`
}
ModuleFieldMappingSetFindType int
)
// Find looks up a origin or destination mapping
func (list *ModuleFieldMappingSet) FindByName(name string, findType ModuleFieldMappingSetFindType) (*ModuleFieldMapping, error) {
for _, mfm := range *list {
switch findType {
case ModuleFieldMappingSetFindTypeOrigin:
if mfm.Origin.Name == name {
return mfm, nil
}
case ModuleFieldMappingSetFindTypeDestination:
if mfm.Destination.Name == name {
return mfm, nil
}
}
}
return nil, nil
}
func (list ModuleFieldMappingSet) Value() (driver.Value, error) {
return json.Marshal(list)
}