From e50472ed8b27348388a4faafd6283b072a22d73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 4 Aug 2020 13:14:09 +0200 Subject: [PATCH] Allow the importer to use legacy ids for future references --- pkg/ngimporter/main.go | 46 +++++++++++++- pkg/ngimporter/types/general.go | 2 + pkg/ngimporter/types/import_node.go | 95 ++++++++++++++++++++++++----- 3 files changed, 128 insertions(+), 15 deletions(-) diff --git a/pkg/ngimporter/main.go b/pkg/ngimporter/main.go index dfe8ce19a..d019a7fe1 100644 --- a/pkg/ngimporter/main.go +++ b/pkg/ngimporter/main.go @@ -38,6 +38,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er imp := &Importer{} db := repository.DB(ctx) modRepo := repository.Module(ctx, db) + recRepo := repository.Record(ctx, db) var err error // import users @@ -50,7 +51,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er } // maps sourceUserID to CortezaID - var uMap map[string]uint64 + uMap := make(map[string]uint64) if usrSrc != nil { um, mgu, err := importUsers(ctx, usrSrc, ns) if err != nil { @@ -73,6 +74,49 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er } } + // populate with existing users + uMod, err := findModuleByHandle(modRepo, ns.ID, "user") + if err != nil { + return err + } + rr, _, err := recRepo.Find(uMod, cct.RecordFilter{ + ModuleID: uMod.ID, + Deleted: rh.FilterStateInclusive, + NamespaceID: ns.ID, + Query: "sys_legacy_ref_id IS NOT NULL", + PageFilter: rh.PageFilter{ + Page: 1, + PerPage: 0, + }, + }) + if err != nil { + return err + } + + rvs, err := recRepo.LoadValues(uMod.Fields.Names(), rr.IDs()) + if err != nil { + return err + } + + err = rr.Walk(func(r *cct.Record) error { + r.Values = rvs.FilterByRecordID(r.ID) + return nil + }) + if err != nil { + return err + } + + rr.Walk(func(r *cct.Record) error { + vr := r.Values.Get("sys_legacy_ref_id", 0) + vu := r.Values.Get("UserID", 0) + u, err := strconv.ParseUint(vu.Value, 10, 64) + if err != nil { + return err + } + uMap[vr.Value] = u + return nil + }) + iss, err = joinData(iss) if err != nil { return err diff --git a/pkg/ngimporter/types/general.go b/pkg/ngimporter/types/general.go index 1e013ab9a..8f5100a70 100644 --- a/pkg/ngimporter/types/general.go +++ b/pkg/ngimporter/types/general.go @@ -21,6 +21,8 @@ const ( MetaMapExt = ".map.json" MetaJoinExt = ".join.json" MetaValueExt = ".value.json" + + LegacyRefIDField = "sys_legacy_ref_id" ) var ( diff --git a/pkg/ngimporter/types/import_node.go b/pkg/ngimporter/types/import_node.go index 561580f71..ca44a2e3c 100644 --- a/pkg/ngimporter/types/import_node.go +++ b/pkg/ngimporter/types/import_node.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "strconv" "strings" "sync" "time" @@ -14,6 +15,7 @@ import ( "github.com/cortezaproject/corteza-server/compose/repository" cv "github.com/cortezaproject/corteza-server/compose/service/values" "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/rh" "github.com/schollz/progressbar/v2" ) @@ -154,6 +156,41 @@ func (n *ImportNode) Import(repoRecord repository.RecordRepository, users map[st } } +func (n *ImportNode) fetchRemoteRef(ref, refMod string, repo repository.RecordRepository) (string, error) { + refModU, err := strconv.ParseUint(refMod, 10, 64) + if err != nil { + return "", err + } + + fl := types.RecordFilter{ + ModuleID: refModU, + NamespaceID: n.Namespace.ID, + Deleted: rh.FilterStateInclusive, + Query: fmt.Sprintf("%s='%s'", LegacyRefIDField, ref), + PageFilter: rh.PageFilter{ + Page: 1, + PerPage: 1, + }, + } + + var refModM *types.Module + if ModulesGlobal != nil { + refModM = ModulesGlobal.FindByID(refModU) + } + if refModM != nil { + rr, _, err := repo.Find(refModM, fl) + if err != nil { + return "", err + } + if len(rr) < 1 { + return "", errors.New(fmt.Sprintf("[error] referenced record %s not found on node %s for module %s", ref, n.Name, refModM.Name)) + } + return strconv.FormatUint(rr[0].ID, 10), nil + } + + return "", nil +} + // determines if node is Satisfied and can be imported // it is Satisfied, when all of it's dependencies have been imported ie. no // more child refs @@ -344,19 +381,26 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { return errors.New("moduleField.record.invalidRefFormat") } + fetch := false + // in case of a missing ref, make sure to remove the reference. // otherwise this will cause internal errors when trying to resolve CortezaID. - if mod, ok := n.idMap[ref]; ok { - if vv, ok := mod[val]; ok { - v.Value = vv - v.Updated = true - } else { - v.Value = "" + if mod, ok := n.idMap[ref]; !ok { + fetch = true + } else if vv, ok := mod[val]; !ok { + fetch = true + } else { + v.Value = vv + v.Updated = true + } + + if fetch { + val, err := n.fetchRemoteRef(val, ref, repo) + if err != nil { continue } - } else { - v.Value = "" - continue + v.Value = val + v.Updated = true } } } @@ -409,6 +453,14 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R recordValues := types.RecordValueSet{} + // assure a valid legacy reference + recordValues = append(recordValues, &types.RecordValue{ + Name: LegacyRefIDField, + Value: record[0], + Place: 0, + Updated: true, + }) + // convert the given row into a { field: value } map; this will be used // for expression evaluation row := map[string]string{} @@ -506,13 +558,28 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R return nil, errors.New("moduleField.record.invalidRefFormat") } - if mod, ok := n.idMap[ref]; ok && val != "" { - if v, ok := mod[val]; ok && v != "" { - val = v - } else { + fetch := false + + if val == "" { + continue + } + + if mod, ok := n.idMap[ref]; !ok { + fetch = true + } else if v, ok := mod[val]; !ok || v == "" { + fetch = true + } else { + val = v + } + + if fetch { + val, err = n.fetchRemoteRef(val, ref, repo) + if err != nil { continue } - } else { + } + + if val == "" { continue } }