From 6652436852c01f45a6eecebc359d4f92667d9a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:14:56 +0200 Subject: [PATCH 1/7] Skip records with no ID --- pkg/ngimporter/types/import_node.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/ngimporter/types/import_node.go b/pkg/ngimporter/types/import_node.go index 3c03f156f..5c96c215a 100644 --- a/pkg/ngimporter/types/import_node.go +++ b/pkg/ngimporter/types/import_node.go @@ -389,6 +389,10 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R return nil, err } + if record[0] == "" { + continue + } + rr := &types.Record{ ModuleID: n.Module.ID, NamespaceID: n.Namespace.ID, From 7ac8479e300ded1cecbd41a68c19906e4ab02020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:16:41 +0200 Subject: [PATCH 2/7] Skip data mapping for missing fields --- pkg/ngimporter/map_data.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/ngimporter/map_data.go b/pkg/ngimporter/map_data.go index ce6014166..f2e196e27 100644 --- a/pkg/ngimporter/map_data.go +++ b/pkg/ngimporter/map_data.go @@ -128,7 +128,10 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) { } } - val := record[hMap[from]] + val := "" + if i, has := hMap[from]; has { + val = record[i] + } // handle data join if strings.Contains(from, ".") { From 305379aeb68d248bcb6b583e484fe6d7c554d869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:17:48 +0200 Subject: [PATCH 3/7] Provide some aditional gval functions --- pkg/ngimporter/types/eval.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pkg/ngimporter/types/eval.go b/pkg/ngimporter/types/eval.go index df74c83ba..2f22ba1f4 100644 --- a/pkg/ngimporter/types/eval.go +++ b/pkg/ngimporter/types/eval.go @@ -11,9 +11,23 @@ import ( // Glang generates a gval language, that can be used for expression evaluation func GLang() gval.Language { return gval.NewLanguage( - gval.JSON(), - gval.Arithmetic(), - gval.PropositionalLogic(), + gval.Full(), + + gval.Function("f64", func(v string) (float64, error) { + nn, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, err + } + return nn, nil + }), + + gval.Function("concat", func(vv ...string) (string, error) { + out := "" + for _, v := range vv { + out += v + } + return out, nil + }), gval.Function("numFmt", func(number, format string) (string, error) { nn, err := strconv.ParseFloat(number, 64) From d19e3059716aef8a622aef87be98dc5024840c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:18:53 +0200 Subject: [PATCH 4/7] Make sys field matching more robust --- pkg/ngimporter/import_users.go | 19 ++++++++++--------- pkg/ngimporter/types/import_node.go | 16 ++++++++-------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pkg/ngimporter/import_users.go b/pkg/ngimporter/import_users.go index 3f9ac3e20..072373723 100644 --- a/pkg/ngimporter/import_users.go +++ b/pkg/ngimporter/import_users.go @@ -5,6 +5,7 @@ import ( "context" "encoding/csv" "io" + "strings" "time" "github.com/cortezaproject/corteza-server/compose/repository" @@ -54,28 +55,28 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace) // when creating users we only care about a handfull of values. // the rest are included in the module - switch h { - case "Username": + switch strings.ToLower(h) { + case "username": u.Username = record[i] break - case "Email": + case "email": u.Email = record[i] break - case "FirstName": + case "firstname": u.Name = record[i] break - case "LastName": + case "lastname": u.Name = u.Name + " " + record[i] break - case "Alias": + case "alias": u.Handle = record[i] break - case "CreatedDate": + case "createddate": if val != "" { u.CreatedAt, err = time.Parse(types.SfDateTimeLayout, val) if err != nil { @@ -84,7 +85,7 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace) } break - case "LastModifiedDate": + case "lastmodifieddate": if val != "" { tt, err := time.Parse(types.SfDateTimeLayout, val) u.UpdatedAt = &tt @@ -95,7 +96,7 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace) break // ignore deleted values, as SF provides minimal info about those - case "IsDeleted": + case "isdeleted": if val == "1" { goto looper } diff --git a/pkg/ngimporter/types/import_node.go b/pkg/ngimporter/types/import_node.go index 5c96c215a..968fc1983 100644 --- a/pkg/ngimporter/types/import_node.go +++ b/pkg/ngimporter/types/import_node.go @@ -415,19 +415,19 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R // system values should be kept on the record's root level if isSysField(h) { - switch h { - case "OwnerId": + switch strings.ToLower(h) { + case "ownerid": rr.OwnedBy = users[val] break // ignore deleted values, as SF provides minimal info about those - case "IsDeleted": - if val == "1" { + case "isdeleted": + if val == "1" || strings.ToLower(val) == "true" { goto looper } break - case "CreatedDate": + case "createddate": if val != "" { rr.CreatedAt, err = time.Parse(SfDateTimeLayout, val) if err != nil { @@ -436,15 +436,15 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R } break - case "CreatedById": + case "createdbyid": rr.CreatedBy = users[val] break - case "LastModifiedById": + case "lastmodifiedbyid": rr.UpdatedBy = users[val] break - case "LastModifiedDate": + case "lastmodifieddate": if val != "" { tt, err := time.Parse(SfDateTimeLayout, val) rr.UpdatedAt = &tt From 573e6082bdddb13dd4b9d712801c1544d4407eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:21:59 +0200 Subject: [PATCH 5/7] Include value sanitizer on imported data Simplifies the entire process and assures data consistency. --- pkg/ngimporter/types/import_node.go | 46 ++++++++++++++++------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/pkg/ngimporter/types/import_node.go b/pkg/ngimporter/types/import_node.go index 968fc1983..561580f71 100644 --- a/pkg/ngimporter/types/import_node.go +++ b/pkg/ngimporter/types/import_node.go @@ -6,12 +6,13 @@ import ( "errors" "fmt" "io" - "strconv" + "log" "strings" "sync" "time" "github.com/cortezaproject/corteza-server/compose/repository" + cv "github.com/cortezaproject/corteza-server/compose/service/values" "github.com/cortezaproject/corteza-server/compose/types" "github.com/schollz/progressbar/v2" ) @@ -318,6 +319,8 @@ func isSysField(f string) bool { // updates the given node's record values that depend on another record func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { + s := cv.Sanitizer() + for _, r := range n.records { for _, v := range r.Values { var f *types.ModuleField @@ -346,6 +349,7 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { if mod, ok := n.idMap[ref]; ok { if vv, ok := mod[val]; ok { v.Value = vv + v.Updated = true } else { v.Value = "" continue @@ -365,10 +369,13 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { } } + nv = s.Run(n.Module, nv) + r.Values = nv err := repo.UpdateValues(r.ID, r.Values) if err != nil { - return err + log.Printf("[issue] db.UpdateValues | %d | %s | %s \n", r.ID, r.Values.String(), err.Error()) + // return err } } return nil @@ -377,6 +384,7 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { // imports the given node's source func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.RecordRepository) (Map, error) { mapping := make(Map) + s := cv.Sanitizer() for { looper: @@ -524,13 +532,6 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R continue } - if f.Kind == "DateTime" { - val, err = assureDateFormat(val, f.Options) - if err != nil { - return nil, err - } - } - values = append(values, val) } } @@ -542,18 +543,19 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R return nil, err } rv := &types.RecordValue{ - Name: h, - Value: v, - Place: uint(i), + Name: h, + Value: v, + Place: uint(i), + Updated: true, } - if f.IsRef() && v != "" { - rv.Ref, err = strconv.ParseUint(v, 10, 64) - if err != nil { - return nil, err - } + // ref values of spliced nodes should get updated later + if n.isSpliced && f.IsRef() { + rv.Updated = false } recordValues = append(recordValues, rv) } + + recordValues = s.Run(n.Module, recordValues) } } @@ -567,9 +569,13 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R for _, v := range recordValues { v.RecordID = r.ID } - err = repo.UpdateValues(r.ID, recordValues) - if err != nil { - return nil, err + + if !n.isSpliced { + err = repo.UpdateValues(r.ID, recordValues) + if err != nil { + log.Printf("[issue] db.UpdateValues | %d | %s | %s \n", r.ID, recordValues.String(), err.Error()) + // return nil, err + } } // spliced nodes should preserve their records for later ref processing From 487bdd80f8d570a49218435c17ce86c3a7e2e795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:22:55 +0200 Subject: [PATCH 6/7] Change SF DateTime layout to RFC standard --- pkg/ngimporter/types/general.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/ngimporter/types/general.go b/pkg/ngimporter/types/general.go index 4a871f161..1e013ab9a 100644 --- a/pkg/ngimporter/types/general.go +++ b/pkg/ngimporter/types/general.go @@ -1,10 +1,13 @@ package types -import "unicode/utf8" +import ( + "time" + "unicode/utf8" +) const ( // SfDateTimeLayout represents the date-time template used by sales force - SfDateTimeLayout = "2006-01-02 15:04:05" + SfDateTimeLayout = time.RFC3339 // DateOnlyLayout represents our internal date only date-time fields DateOnlyLayout = "2006-01-02" // TimeOnlyLayout represents our internal time only date-time fields From 11fe268275d8237aaddedf2a5c89ea352f44314d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Jerman?= Date: Tue, 14 Jul 2020 12:23:53 +0200 Subject: [PATCH 7/7] Move module fetching to the DB level --- pkg/ngimporter/main.go | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/pkg/ngimporter/main.go b/pkg/ngimporter/main.go index 8212befed..dfe8ce19a 100644 --- a/pkg/ngimporter/main.go +++ b/pkg/ngimporter/main.go @@ -9,7 +9,6 @@ import ( "sync" "github.com/cortezaproject/corteza-server/compose/repository" - "github.com/cortezaproject/corteza-server/compose/service" cct "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" "github.com/schollz/progressbar/v2" @@ -37,7 +36,8 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er // contains warnings raised by the pre process steps var preProcW []string imp := &Importer{} - svcMod := service.DefaultModule.With(ctx) + db := repository.DB(ctx) + modRepo := repository.Module(ctx, db) var err error // import users @@ -87,7 +87,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er for _, nIs := range nIss { // preload module - mod, err := svcMod.FindByHandle(ns.ID, nIs.Name) + mod, err := findModuleByHandle(modRepo, ns.ID, nIs.Name) if err != nil { preProcW = append(preProcW, err.Error()+" "+nIs.Name) continue @@ -142,7 +142,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er continue } - mm, err := svcMod.FindByID(ns.ID, vv) + mm, err := findModuleByID(modRepo, ns.ID, vv) if err != nil { preProcW = append(preProcW, err.Error()+" "+nIs.Name+" "+f.Name+" "+modID) continue @@ -274,3 +274,31 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error { return nil }) } + +func findModuleByID(repo repository.ModuleRepository, namespaceID, moduleID uint64) (*cct.Module, error) { + var err error + mod, err := repo.FindByID(namespaceID, moduleID) + if err != nil { + return nil, err + } + mod.Fields, err = repo.FindFields(mod.ID) + if err != nil { + return nil, err + } + + return mod, nil +} + +func findModuleByHandle(repo repository.ModuleRepository, namespaceID uint64, handle string) (*cct.Module, error) { + var err error + mod, err := repo.FindByHandle(namespaceID, handle) + if err != nil { + return nil, err + } + mod.Fields, err = repo.FindFields(mod.ID) + if err != nil { + return nil, err + } + + return mod, nil +}