From 6ccc4c7294ffadbf76dc1b6e1d91fb08e3848400 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Toma=C5=BE=20Jerman?= <tomaz.jerman@kendu.si>
Date: Mon, 23 Mar 2020 21:36:40 +0100
Subject: [PATCH] Add support for per field value mapping

---
 compose/commands/migrator.go     | 23 +++++++++++++++++++++++
 pkg/migrate/README.adoc          | 32 ++++++++++++++++++++++++++++++++
 pkg/migrate/main.go              |  1 +
 pkg/migrate/stream.go            |  1 +
 pkg/migrate/types/migrateable.go |  4 ++++
 pkg/migrate/types/node.go        | 14 ++++++++++++++
 6 files changed, 75 insertions(+)

diff --git a/compose/commands/migrator.go b/compose/commands/migrator.go
index f3bfcf5ef..18fe5ee15 100644
--- a/compose/commands/migrator.go
+++ b/compose/commands/migrator.go
@@ -1,6 +1,8 @@
 package commands
 
 import (
+	"encoding/json"
+	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -98,6 +100,27 @@ func Migrator() *cobra.Command {
 					mm.Name = name
 					mm.Join = file
 
+					mg = migrateableAdd(mg, mm)
+				} else if strings.HasSuffix(info.Name(), ".value.json") {
+					file, err := os.Open(path)
+					if err != nil {
+						log.Fatal(err)
+					}
+
+					ext := filepath.Ext(info.Name())
+					// @todo improve this!!
+					name := info.Name()[0 : len(info.Name())-len(ext)-6]
+					mm := migrateableSource(mg, name)
+					mm.Name = name
+
+					var vmp map[string]map[string]string
+					src, _ := ioutil.ReadAll(file)
+					err = json.Unmarshal(src, &vmp)
+					if err != nil {
+						log.Fatal(err)
+					}
+					mm.ValueMap = vmp
+
 					mg = migrateableAdd(mg, mm)
 				}
 				return nil
diff --git a/pkg/migrate/README.adoc b/pkg/migrate/README.adoc
index 451490df6..112352ecc 100644
--- a/pkg/migrate/README.adoc
+++ b/pkg/migrate/README.adoc
@@ -129,3 +129,35 @@ When creating a `.map.json` file, values from the join operation are available u
   }
 ]
 ----
+
+== Value Mapping
+
+The system allows us to map a specific value from the provided `.csv` file into a value used by the system.
+For example; we can map `In Progress` into `in_progress`.
+The mapping also supports a default value, by using the `*` wildcard.
+
+=== Algorithrm
+
+* unmarshal the given `.value.json`
+* before applying a value for the given field, attempt to map the value
+** if mapping is successful, use the mapped value,
+** else if default value exists, use the default value,
+** else use the original value.
+
+=== Example
+
+.source.values.json
+
+The following value mapping maps `sys_status` field's values; the left one into the right one, with a default of `"new"` (`"*": "new"`).
+
+[source,json]
+----
+{
+  "sys_status": {
+    "In Progress": "in_progress",
+    "Send to QA": "qa_pending",
+    "Submit Job": "qa_approved",
+    "*": "new"
+  }
+}
+----
diff --git a/pkg/migrate/main.go b/pkg/migrate/main.go
index 3abf05daa..5c6cd7ead 100644
--- a/pkg/migrate/main.go
+++ b/pkg/migrate/main.go
@@ -117,6 +117,7 @@ func Migrate(mg []types.Migrateable, ns *cct.Namespace, ctx context.Context) err
 				Header:    header,
 				Lock:      &sync.Mutex{},
 				FieldMap:  m.FieldMap,
+				ValueMap:  m.ValueMap,
 			}
 			n = mig.AddNode(n)
 
diff --git a/pkg/migrate/stream.go b/pkg/migrate/stream.go
index c6c4bc59e..1407badfe 100644
--- a/pkg/migrate/stream.go
+++ b/pkg/migrate/stream.go
@@ -162,6 +162,7 @@ func splitStream(m types.Migrateable) ([]types.Migrateable, error) {
 			Header:   &v.header,
 			FieldMap: m.FieldMap,
 			AliasMap: m.AliasMap,
+			ValueMap: m.ValueMap,
 		})
 	}
 
diff --git a/pkg/migrate/types/migrateable.go b/pkg/migrate/types/migrateable.go
index 5dde2bb39..a3d4a14e7 100644
--- a/pkg/migrate/types/migrateable.go
+++ b/pkg/migrate/types/migrateable.go
@@ -29,5 +29,9 @@ type (
 		FieldMap map[string]JoinedNodeRecords
 		// helps us determine what value field to use for linking
 		AliasMap map[string]string
+
+		// value is used for field value mapping
+		// field: value from: value to
+		ValueMap map[string]map[string]string
 	}
 )
diff --git a/pkg/migrate/types/node.go b/pkg/migrate/types/node.go
index 17785228f..2252bc9c5 100644
--- a/pkg/migrate/types/node.go
+++ b/pkg/migrate/types/node.go
@@ -57,6 +57,9 @@ type (
 
 		// field: recordID: [value]
 		FieldMap map[string]JoinedNodeRecords
+
+		// field: value from: value to
+		ValueMap map[string]map[string]string
 	}
 
 	// map between migrated ID and Corteza ID
@@ -173,6 +176,9 @@ func (n *Node) Merge(nn *Node) {
 	if nn.FieldMap != nil {
 		n.FieldMap = nn.FieldMap
 	}
+	if nn.ValueMap != nil {
+		n.ValueMap = nn.ValueMap
+	}
 }
 
 // link the two nodes
@@ -519,6 +525,14 @@ func importNodeSource(n *Node, users map[string]uint64, repo repository.RecordRe
 				}
 
 				for i, v := range values {
+					if fmp, ok := n.ValueMap[h]; ok {
+						if mpv, ok := fmp[v]; ok {
+							v = mpv
+						} else if mpv, ok := fmp["*"]; ok {
+							v = mpv
+						}
+					}
+
 					vals = append(vals, &types.RecordValue{
 						Name:  h,
 						Value: v,