From 8e31d43c294f2da5b9a14b04431d6199249f3da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 24 Mar 2020 11:00:46 +0100 Subject: [PATCH] Add support for composite keys with join opp. --- pkg/migrate/README.adoc | 10 +++++ pkg/migrate/join.go | 72 ++++++++++++++++++++++---------- pkg/migrate/stream.go | 7 +++- pkg/migrate/types/migrateable.go | 2 +- 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/pkg/migrate/README.adoc b/pkg/migrate/README.adoc index 112352ecc..60844a0d0 100644 --- a/pkg/migrate/README.adoc +++ b/pkg/migrate/README.adoc @@ -130,6 +130,16 @@ When creating a `.map.json` file, values from the join operation are available u ] ---- +It is also possible to define a join operation on multiple fields at the same time -- useful in cases where a unique PK is not available and must be constructed. +The following example uses `CreatedDate` and `CreatedById` fields as an index. + +[source,json] +---- +{ + "[CreatedDate,CreatedById]->smod": "subMod.[CreatedDate,CreatedById]" +} +---- + == Value Mapping The system allows us to map a specific value from the provided `.csv` file into a value used by the system. diff --git a/pkg/migrate/join.go b/pkg/migrate/join.go index cc799b2c2..9a75a1b6a 100644 --- a/pkg/migrate/join.go +++ b/pkg/migrate/join.go @@ -6,6 +6,7 @@ import ( "errors" "io" "io/ioutil" + "regexp" "strings" "github.com/cortezaproject/corteza-server/pkg/migrate/types" @@ -17,11 +18,11 @@ type ( mapLink struct { jn *types.JoinedNode // field from the base node used in the op. - baseField string + baseField []string // alias to use for the base field; allows us to use the same field multiple times baseFieldAlias string // field from the joined node to use in the opp. - joinField string + joinField []string } // temporary node for the join op. @@ -29,7 +30,14 @@ type ( mg *types.Migrateable // temporary migration node mapper based on aliases mapper map[string]mapLink - aliasMap map[string]string + aliasMap map[string][]string + } + + exprEval struct { + baseFields []string + baseFieldAlias string + joinModule string + joinFields []string } ) @@ -57,7 +65,7 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) { // defer this, so we can do simple nil checks nd.mapper = make(map[string]mapLink) - nd.aliasMap = make(map[string]string) + nd.aliasMap = make(map[string][]string) // join definition map defines how two sources are joined var joinDef map[string]string @@ -69,37 +77,31 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) { // find all joined nodes for the given base node for base, condition := range joinDef { - ptsB := strings.Split(base, "->") - baseField := ptsB[0] - baseFieldAlias := ptsB[1] + expr := splitExpr(base, condition) - if _, ok := nd.aliasMap[baseFieldAlias]; ok { - return nil, errors.New("alias.used " + nd.mg.Name + " " + baseFieldAlias) + if _, ok := nd.aliasMap[expr.baseFieldAlias]; ok { + return nil, errors.New("alias.used " + nd.mg.Name + " " + expr.baseFieldAlias) } - nd.aliasMap[baseFieldAlias] = baseField - - ptsC := strings.Split(condition, ".") - joinedModule := ptsC[0] - joinedField := ptsC[1] + nd.aliasMap[expr.baseFieldAlias] = expr.baseFields // register migration node as join node for _, m := range mm { - if m.Name == joinedModule { - if _, ok := joinedNodes[joinedModule]; !ok { + if m.Name == expr.joinModule { + if _, ok := joinedNodes[expr.joinModule]; !ok { ww := m - joinedNodes[joinedModule] = &types.JoinedNode{ + joinedNodes[expr.joinModule] = &types.JoinedNode{ Mg: &ww, Name: ww.Name, } } // create a link between the base and joined node - jn := joinedNodes[joinedModule] - nd.mapper[baseFieldAlias] = mapLink{ + jn := joinedNodes[expr.joinModule] + nd.mapper[expr.baseFieldAlias] = mapLink{ jn: jn, - baseField: baseField, - baseFieldAlias: baseFieldAlias, - joinField: joinedField, + baseField: expr.baseFields, + baseFieldAlias: expr.baseFieldAlias, + joinField: expr.joinFields, } break } @@ -168,7 +170,12 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) { } for _, e := range link.jn.Entries { - kk := alias + "." + e[link.joinField] + jj := []string{} + for _, jf := range link.joinField { + jj = append(jj, e[jf]) + } + + kk := alias + "." + strings.Join(jj[:], ".") if _, ok := o.FieldMap[kk]; !ok { o.FieldMap[kk] = make(types.JoinedNodeRecords, 0) } @@ -182,3 +189,22 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) { return out, nil } + +// helper to split the join expression +func splitExpr(base, joined string) exprEval { + rr := exprEval{} + + // original node + rx := regexp.MustCompile(`\[?(?P[\w,]+)\]?->(?P\w+)`) + mx := rx.FindStringSubmatch(base) + rr.baseFields = strings.Split(mx[1], ",") + rr.baseFieldAlias = mx[2] + + // joined node + rx = regexp.MustCompile(`(?P\w+)\.\[?(?P[\w,]+)\]?`) + mx = rx.FindStringSubmatch(joined) + rr.joinModule = mx[1] + rr.joinFields = strings.Split(mx[2], ",") + + return rr +} diff --git a/pkg/migrate/stream.go b/pkg/migrate/stream.go index eb644b108..555e99bc8 100644 --- a/pkg/migrate/stream.go +++ b/pkg/migrate/stream.go @@ -131,7 +131,12 @@ func splitStream(m types.Migrateable) ([]types.Migrateable, error) { baseFieldAlias := pts[0] originalOn := m.AliasMap[baseFieldAlias] joinField := pts[1] - val = baseFieldAlias + "." + record[hMap[originalOn]] + + oo := []string{} + for _, ff := range originalOn { + oo = append(oo, record[hMap[ff]]) + } + val = baseFieldAlias + "." + strings.Join(oo[:], ".") // modify header field to specify what joined node field to use nmF += ":" + joinField diff --git a/pkg/migrate/types/migrateable.go b/pkg/migrate/types/migrateable.go index a3d4a14e7..0b02a9bf1 100644 --- a/pkg/migrate/types/migrateable.go +++ b/pkg/migrate/types/migrateable.go @@ -28,7 +28,7 @@ type ( // alias.ID: [value] FieldMap map[string]JoinedNodeRecords // helps us determine what value field to use for linking - AliasMap map[string]string + AliasMap map[string][]string // value is used for field value mapping // field: value from: value to