diff --git a/pkg/ngimporter/import_users.go b/pkg/ngimporter/import_users.go index 3f9ac3e20..072373723 100644 --- a/pkg/ngimporter/import_users.go +++ b/pkg/ngimporter/import_users.go @@ -5,6 +5,7 @@ import ( "context" "encoding/csv" "io" + "strings" "time" "github.com/cortezaproject/corteza-server/compose/repository" @@ -54,28 +55,28 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace) // when creating users we only care about a handfull of values. // the rest are included in the module - switch h { - case "Username": + switch strings.ToLower(h) { + case "username": u.Username = record[i] break - case "Email": + case "email": u.Email = record[i] break - case "FirstName": + case "firstname": u.Name = record[i] break - case "LastName": + case "lastname": u.Name = u.Name + " " + record[i] break - case "Alias": + case "alias": u.Handle = record[i] break - case "CreatedDate": + case "createddate": if val != "" { u.CreatedAt, err = time.Parse(types.SfDateTimeLayout, val) if err != nil { @@ -84,7 +85,7 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace) } break - case "LastModifiedDate": + case "lastmodifieddate": if val != "" { tt, err := time.Parse(types.SfDateTimeLayout, val) u.UpdatedAt = &tt @@ -95,7 +96,7 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace) break // ignore deleted values, as SF provides minimal info about those - case "IsDeleted": + case "isdeleted": if val == "1" { goto looper } diff --git a/pkg/ngimporter/main.go b/pkg/ngimporter/main.go index 8212befed..dfe8ce19a 100644 --- a/pkg/ngimporter/main.go +++ b/pkg/ngimporter/main.go @@ -9,7 +9,6 @@ import ( "sync" "github.com/cortezaproject/corteza-server/compose/repository" - "github.com/cortezaproject/corteza-server/compose/service" cct "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" "github.com/schollz/progressbar/v2" @@ -37,7 +36,8 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er // contains warnings raised by the pre process steps var preProcW []string imp := &Importer{} - svcMod := service.DefaultModule.With(ctx) + db := repository.DB(ctx) + modRepo := repository.Module(ctx, db) var err error // import users @@ -87,7 +87,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er for _, nIs := range nIss { // preload module - mod, err := svcMod.FindByHandle(ns.ID, nIs.Name) + mod, err := findModuleByHandle(modRepo, ns.ID, nIs.Name) if err != nil { preProcW = append(preProcW, err.Error()+" "+nIs.Name) continue @@ -142,7 +142,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er continue } - mm, err := svcMod.FindByID(ns.ID, vv) + mm, err := findModuleByID(modRepo, ns.ID, vv) if err != nil { preProcW = append(preProcW, err.Error()+" "+nIs.Name+" "+f.Name+" "+modID) continue @@ -274,3 +274,31 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error { return nil }) } + +func findModuleByID(repo repository.ModuleRepository, namespaceID, moduleID uint64) (*cct.Module, error) { + var err error + mod, err := repo.FindByID(namespaceID, moduleID) + if err != nil { + return nil, err + } + mod.Fields, err = repo.FindFields(mod.ID) + if err != nil { + return nil, err + } + + return mod, nil +} + +func findModuleByHandle(repo repository.ModuleRepository, namespaceID uint64, handle string) (*cct.Module, error) { + var err error + mod, err := repo.FindByHandle(namespaceID, handle) + if err != nil { + return nil, err + } + mod.Fields, err = repo.FindFields(mod.ID) + if err != nil { + return nil, err + } + + return mod, nil +} diff --git a/pkg/ngimporter/map_data.go b/pkg/ngimporter/map_data.go index ce6014166..f2e196e27 100644 --- a/pkg/ngimporter/map_data.go +++ b/pkg/ngimporter/map_data.go @@ -128,7 +128,10 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) { } } - val := record[hMap[from]] + val := "" + if i, has := hMap[from]; has { + val = record[i] + } // handle data join if strings.Contains(from, ".") { diff --git a/pkg/ngimporter/types/eval.go b/pkg/ngimporter/types/eval.go index df74c83ba..2f22ba1f4 100644 --- a/pkg/ngimporter/types/eval.go +++ b/pkg/ngimporter/types/eval.go @@ -11,9 +11,23 @@ import ( // Glang generates a gval language, that can be used for expression evaluation func GLang() gval.Language { return gval.NewLanguage( - gval.JSON(), - gval.Arithmetic(), - gval.PropositionalLogic(), + gval.Full(), + + gval.Function("f64", func(v string) (float64, error) { + nn, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0, err + } + return nn, nil + }), + + gval.Function("concat", func(vv ...string) (string, error) { + out := "" + for _, v := range vv { + out += v + } + return out, nil + }), gval.Function("numFmt", func(number, format string) (string, error) { nn, err := strconv.ParseFloat(number, 64) diff --git a/pkg/ngimporter/types/general.go b/pkg/ngimporter/types/general.go index 4a871f161..1e013ab9a 100644 --- a/pkg/ngimporter/types/general.go +++ b/pkg/ngimporter/types/general.go @@ -1,10 +1,13 @@ package types -import "unicode/utf8" +import ( + "time" + "unicode/utf8" +) const ( // SfDateTimeLayout represents the date-time template used by sales force - SfDateTimeLayout = "2006-01-02 15:04:05" + SfDateTimeLayout = time.RFC3339 // DateOnlyLayout represents our internal date only date-time fields DateOnlyLayout = "2006-01-02" // TimeOnlyLayout represents our internal time only date-time fields diff --git a/pkg/ngimporter/types/import_node.go b/pkg/ngimporter/types/import_node.go index 3c03f156f..561580f71 100644 --- a/pkg/ngimporter/types/import_node.go +++ b/pkg/ngimporter/types/import_node.go @@ -6,12 +6,13 @@ import ( "errors" "fmt" "io" - "strconv" + "log" "strings" "sync" "time" "github.com/cortezaproject/corteza-server/compose/repository" + cv "github.com/cortezaproject/corteza-server/compose/service/values" "github.com/cortezaproject/corteza-server/compose/types" "github.com/schollz/progressbar/v2" ) @@ -318,6 +319,8 @@ func isSysField(f string) bool { // updates the given node's record values that depend on another record func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { + s := cv.Sanitizer() + for _, r := range n.records { for _, v := range r.Values { var f *types.ModuleField @@ -346,6 +349,7 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { if mod, ok := n.idMap[ref]; ok { if vv, ok := mod[val]; ok { v.Value = vv + v.Updated = true } else { v.Value = "" continue @@ -365,10 +369,13 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { } } + nv = s.Run(n.Module, nv) + r.Values = nv err := repo.UpdateValues(r.ID, r.Values) if err != nil { - return err + log.Printf("[issue] db.UpdateValues | %d | %s | %s \n", r.ID, r.Values.String(), err.Error()) + // return err } } return nil @@ -377,6 +384,7 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error { // imports the given node's source func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.RecordRepository) (Map, error) { mapping := make(Map) + s := cv.Sanitizer() for { looper: @@ -389,6 +397,10 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R return nil, err } + if record[0] == "" { + continue + } + rr := &types.Record{ ModuleID: n.Module.ID, NamespaceID: n.Namespace.ID, @@ -411,19 +423,19 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R // system values should be kept on the record's root level if isSysField(h) { - switch h { - case "OwnerId": + switch strings.ToLower(h) { + case "ownerid": rr.OwnedBy = users[val] break // ignore deleted values, as SF provides minimal info about those - case "IsDeleted": - if val == "1" { + case "isdeleted": + if val == "1" || strings.ToLower(val) == "true" { goto looper } break - case "CreatedDate": + case "createddate": if val != "" { rr.CreatedAt, err = time.Parse(SfDateTimeLayout, val) if err != nil { @@ -432,15 +444,15 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R } break - case "CreatedById": + case "createdbyid": rr.CreatedBy = users[val] break - case "LastModifiedById": + case "lastmodifiedbyid": rr.UpdatedBy = users[val] break - case "LastModifiedDate": + case "lastmodifieddate": if val != "" { tt, err := time.Parse(SfDateTimeLayout, val) rr.UpdatedAt = &tt @@ -520,13 +532,6 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R continue } - if f.Kind == "DateTime" { - val, err = assureDateFormat(val, f.Options) - if err != nil { - return nil, err - } - } - values = append(values, val) } } @@ -538,18 +543,19 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R return nil, err } rv := &types.RecordValue{ - Name: h, - Value: v, - Place: uint(i), + Name: h, + Value: v, + Place: uint(i), + Updated: true, } - if f.IsRef() && v != "" { - rv.Ref, err = strconv.ParseUint(v, 10, 64) - if err != nil { - return nil, err - } + // ref values of spliced nodes should get updated later + if n.isSpliced && f.IsRef() { + rv.Updated = false } recordValues = append(recordValues, rv) } + + recordValues = s.Run(n.Module, recordValues) } } @@ -563,9 +569,13 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R for _, v := range recordValues { v.RecordID = r.ID } - err = repo.UpdateValues(r.ID, recordValues) - if err != nil { - return nil, err + + if !n.isSpliced { + err = repo.UpdateValues(r.ID, recordValues) + if err != nil { + log.Printf("[issue] db.UpdateValues | %d | %s | %s \n", r.ID, recordValues.String(), err.Error()) + // return nil, err + } } // spliced nodes should preserve their records for later ref processing