Add support for composite keys with join opp.

pkg/migrate/README.adoc

@@ -130,6 +130,16 @@ When creating a `.map.json` file, values from the join operation are available u
 ]
 ----
 
+It is also possible to define a join operation on multiple fields at the same time -- useful in cases where a unique PK is not available and must be constructed.
+The following example uses `CreatedDate` and `CreatedById` fields as an index.
+
+[source,json]
+----
+{
+  "[CreatedDate,CreatedById]->smod": "subMod.[CreatedDate,CreatedById]"
+}
+----
+
 == Value Mapping
 
 The system allows us to map a specific value from the provided `.csv` file into a value used by the system.

pkg/migrate/join.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"regexp"
 	"strings"
 
 	"github.com/cortezaproject/corteza-server/pkg/migrate/types"
@@ -17,11 +18,11 @@ type (
 	mapLink struct {
 		jn *types.JoinedNode
 		// field from the base node used in the op.
-		baseField string
+		baseField []string
 		// alias to use for the base field; allows us to use the same field multiple times
 		baseFieldAlias string
 		// field from the joined node to use in the opp.
-		joinField string
+		joinField []string
 	}
 
 	// temporary node for the join op.
@@ -29,7 +30,14 @@ type (
 		mg *types.Migrateable
 		// temporary migration node mapper based on aliases
 		mapper   map[string]mapLink
-		aliasMap map[string]string
+		aliasMap map[string][]string
+	}
+
+	exprEval struct {
+		baseFields     []string
+		baseFieldAlias string
+		joinModule     string
+		joinFields     []string
 	}
 )
 
@@ -57,7 +65,7 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) {
 
 		// defer this, so we can do simple nil checks
 		nd.mapper = make(map[string]mapLink)
-		nd.aliasMap = make(map[string]string)
+		nd.aliasMap = make(map[string][]string)
 
 		// join definition map defines how two sources are joined
 		var joinDef map[string]string
@@ -69,37 +77,31 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) {
 
 		// find all joined nodes for the given base node
 		for base, condition := range joinDef {
-			ptsB := strings.Split(base, "->")
-			baseField := ptsB[0]
-			baseFieldAlias := ptsB[1]
+			expr := splitExpr(base, condition)
 
-			if _, ok := nd.aliasMap[baseFieldAlias]; ok {
-				return nil, errors.New("alias.used " + nd.mg.Name + " " + baseFieldAlias)
+			if _, ok := nd.aliasMap[expr.baseFieldAlias]; ok {
+				return nil, errors.New("alias.used " + nd.mg.Name + " " + expr.baseFieldAlias)
 			}
-			nd.aliasMap[baseFieldAlias] = baseField
-
-			ptsC := strings.Split(condition, ".")
-			joinedModule := ptsC[0]
-			joinedField := ptsC[1]
+			nd.aliasMap[expr.baseFieldAlias] = expr.baseFields
 
 			// register migration node as join node
 			for _, m := range mm {
-				if m.Name == joinedModule {
-					if _, ok := joinedNodes[joinedModule]; !ok {
+				if m.Name == expr.joinModule {
+					if _, ok := joinedNodes[expr.joinModule]; !ok {
 						ww := m
-						joinedNodes[joinedModule] = &types.JoinedNode{
+						joinedNodes[expr.joinModule] = &types.JoinedNode{
 							Mg:   &ww,
 							Name: ww.Name,
 						}
 					}
 
 					// create a link between the base and joined node
-					jn := joinedNodes[joinedModule]
-					nd.mapper[baseFieldAlias] = mapLink{
+					jn := joinedNodes[expr.joinModule]
+					nd.mapper[expr.baseFieldAlias] = mapLink{
 						jn:             jn,
-						baseField:      baseField,
-						baseFieldAlias: baseFieldAlias,
-						joinField:      joinedField,
+						baseField:      expr.baseFields,
+						baseFieldAlias: expr.baseFieldAlias,
+						joinField:      expr.joinFields,
 					}
 					break
 				}
@@ -168,7 +170,12 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) {
 			}
 
 			for _, e := range link.jn.Entries {
-				kk := alias + "." + e[link.joinField]
+				jj := []string{}
+				for _, jf := range link.joinField {
+					jj = append(jj, e[jf])
+				}
+
+				kk := alias + "." + strings.Join(jj[:], ".")
 				if _, ok := o.FieldMap[kk]; !ok {
 					o.FieldMap[kk] = make(types.JoinedNodeRecords, 0)
 				}
@@ -182,3 +189,22 @@ func sourceJoin(mm []types.Migrateable) ([]types.Migrateable, error) {
 
 	return out, nil
 }
+
+// helper to split the join expression
+func splitExpr(base, joined string) exprEval {
+	rr := exprEval{}
+
+	// original node
+	rx := regexp.MustCompile(`\[?(?P<bf>[\w,]+)\]?->(?P<bfa>\w+)`)
+	mx := rx.FindStringSubmatch(base)
+	rr.baseFields = strings.Split(mx[1], ",")
+	rr.baseFieldAlias = mx[2]
+
+	// joined node
+	rx = regexp.MustCompile(`(?P<jm>\w+)\.\[?(?P<jmf>[\w,]+)\]?`)
+	mx = rx.FindStringSubmatch(joined)
+	rr.joinModule = mx[1]
+	rr.joinFields = strings.Split(mx[2], ",")
+
+	return rr
+}

pkg/migrate/stream.go

@@ -131,7 +131,12 @@ func splitStream(m types.Migrateable) ([]types.Migrateable, error) {
 						baseFieldAlias := pts[0]
 						originalOn := m.AliasMap[baseFieldAlias]
 						joinField := pts[1]
-						val = baseFieldAlias + "." + record[hMap[originalOn]]
+
+						oo := []string{}
+						for _, ff := range originalOn {
+							oo = append(oo, record[hMap[ff]])
+						}
+						val = baseFieldAlias + "." + strings.Join(oo[:], ".")
 
 						// modify header field to specify what joined node field to use
 						nmF += ":" + joinField

pkg/migrate/types/migrateable.go

@@ -28,7 +28,7 @@ type (
 		// alias.ID: [value]
 		FieldMap map[string]JoinedNodeRecords
 		// helps us determine what value field to use for linking
-		AliasMap map[string]string
+		AliasMap map[string][]string
 
 		// value is used for field value mapping
 		// field: value from: value to