Allow the importer to use legacy ids for future references

pkg/ngimporter/main.go

@@ -38,6 +38,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er
 	imp := &Importer{}
 	db := repository.DB(ctx)
 	modRepo := repository.Module(ctx, db)
+	recRepo := repository.Record(ctx, db)
 	var err error
 
 	// import users
@@ -50,7 +51,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er
 	}
 
 	// maps sourceUserID to CortezaID
-	var uMap map[string]uint64
+	uMap := make(map[string]uint64)
 	if usrSrc != nil {
 		um, mgu, err := importUsers(ctx, usrSrc, ns)
 		if err != nil {
@@ -73,6 +74,49 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er
 		}
 	}
 
+	// populate with existing users
+	uMod, err := findModuleByHandle(modRepo, ns.ID, "user")
+	if err != nil {
+		return err
+	}
+	rr, _, err := recRepo.Find(uMod, cct.RecordFilter{
+		ModuleID:    uMod.ID,
+		Deleted:     rh.FilterStateInclusive,
+		NamespaceID: ns.ID,
+		Query:       "sys_legacy_ref_id IS NOT NULL",
+		PageFilter: rh.PageFilter{
+			Page:    1,
+			PerPage: 0,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	rvs, err := recRepo.LoadValues(uMod.Fields.Names(), rr.IDs())
+	if err != nil {
+		return err
+	}
+
+	err = rr.Walk(func(r *cct.Record) error {
+		r.Values = rvs.FilterByRecordID(r.ID)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	rr.Walk(func(r *cct.Record) error {
+		vr := r.Values.Get("sys_legacy_ref_id", 0)
+		vu := r.Values.Get("UserID", 0)
+		u, err := strconv.ParseUint(vu.Value, 10, 64)
+		if err != nil {
+			return err
+		}
+		uMap[vr.Value] = u
+		return nil
+	})
+
 	iss, err = joinData(iss)
 	if err != nil {
 		return err

pkg/ngimporter/types/general.go

@@ -21,6 +21,8 @@ const (
 	MetaMapExt   = ".map.json"
 	MetaJoinExt  = ".join.json"
 	MetaValueExt = ".value.json"
+
+	LegacyRefIDField = "sys_legacy_ref_id"
 )
 
 var (

pkg/ngimporter/types/import_node.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -14,6 +15,7 @@ import (
 	"github.com/cortezaproject/corteza-server/compose/repository"
 	cv "github.com/cortezaproject/corteza-server/compose/service/values"
 	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/cortezaproject/corteza-server/pkg/rh"
 	"github.com/schollz/progressbar/v2"
 )
 
@@ -154,6 +156,41 @@ func (n *ImportNode) Import(repoRecord repository.RecordRepository, users map[st
 	}
 }
 
+func (n *ImportNode) fetchRemoteRef(ref, refMod string, repo repository.RecordRepository) (string, error) {
+	refModU, err := strconv.ParseUint(refMod, 10, 64)
+	if err != nil {
+		return "", err
+	}
+
+	fl := types.RecordFilter{
+		ModuleID:    refModU,
+		NamespaceID: n.Namespace.ID,
+		Deleted:     rh.FilterStateInclusive,
+		Query:       fmt.Sprintf("%s='%s'", LegacyRefIDField, ref),
+		PageFilter: rh.PageFilter{
+			Page:    1,
+			PerPage: 1,
+		},
+	}
+
+	var refModM *types.Module
+	if ModulesGlobal != nil {
+		refModM = ModulesGlobal.FindByID(refModU)
+	}
+	if refModM != nil {
+		rr, _, err := repo.Find(refModM, fl)
+		if err != nil {
+			return "", err
+		}
+		if len(rr) < 1 {
+			return "", errors.New(fmt.Sprintf("[error] referenced record %s not found on node %s for module %s", ref, n.Name, refModM.Name))
+		}
+		return strconv.FormatUint(rr[0].ID, 10), nil
+	}
+
+	return "", nil
+}
+
 // determines if node is Satisfied and can be imported
 // it is Satisfied, when all of it's dependencies have been imported ie. no
 // more child refs
@@ -344,19 +381,26 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error {
 					return errors.New("moduleField.record.invalidRefFormat")
 				}
 
+				fetch := false
+
 				// in case of a missing ref, make sure to remove the reference.
 				// otherwise this will cause internal errors when trying to resolve CortezaID.
-				if mod, ok := n.idMap[ref]; ok {
-					if vv, ok := mod[val]; ok {
-						v.Value = vv
-						v.Updated = true
-					} else {
-						v.Value = ""
+				if mod, ok := n.idMap[ref]; !ok {
+					fetch = true
+				} else if vv, ok := mod[val]; !ok {
+					fetch = true
+				} else {
+					v.Value = vv
+					v.Updated = true
+				}
+
+				if fetch {
+					val, err := n.fetchRemoteRef(val, ref, repo)
+					if err != nil {
 						continue
 					}
-				} else {
-					v.Value = ""
-					continue
+					v.Value = val
+					v.Updated = true
 				}
 			}
 		}
@@ -409,6 +453,14 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
 
 		recordValues := types.RecordValueSet{}
 
+		// assure a valid legacy reference
+		recordValues = append(recordValues, &types.RecordValue{
+			Name:    LegacyRefIDField,
+			Value:   record[0],
+			Place:   0,
+			Updated: true,
+		})
+
 		// convert the given row into a { field: value } map; this will be used
 		// for expression evaluation
 		row := map[string]string{}
@@ -506,13 +558,28 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
 								return nil, errors.New("moduleField.record.invalidRefFormat")
 							}
 
-							if mod, ok := n.idMap[ref]; ok && val != "" {
-								if v, ok := mod[val]; ok && v != "" {
-									val = v
-								} else {
+							fetch := false
+
+							if val == "" {
+								continue
+							}
+
+							if mod, ok := n.idMap[ref]; !ok {
+								fetch = true
+							} else if v, ok := mod[val]; !ok || v == "" {
+								fetch = true
+							} else {
+								val = v
+							}
+
+							if fetch {
+								val, err = n.fetchRemoteRef(val, ref, repo)
+								if err != nil {
 									continue
 								}
-							} else {
+							}
+
+							if val == "" {
 								continue
 							}
 						}