DeforaNetworks/corteza
DeforaNetworks/corteza
3
0
Fork 0
Code

Merge branch 'feature-importer-tweaks' into develop

Browse Source
This commit is contained in:
Tomaž Jerman 2020-07-14 13:01:13 +02:00
commit 6fa55771fb
6 changed files with 106 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/ngimporter/import_users.go
View File

@ -5,6 +5,7 @@ import (
"context"
"encoding/csv"
"io"
"strings"
"time"
"github.com/cortezaproject/corteza-server/compose/repository"
@ -54,28 +55,28 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace)
// when creating users we only care about a handfull of values.
// the rest are included in the module
switch h {
case "Username":
switch strings.ToLower(h) {
case "username":
u.Username = record[i]
break
case "Email":
case "email":
u.Email = record[i]
break
case "FirstName":
case "firstname":
u.Name = record[i]
break
case "LastName":
case "lastname":
u.Name = u.Name + " " + record[i]
break
case "Alias":
case "alias":
u.Handle = record[i]
break
case "CreatedDate":
case "createddate":
if val != "" {
u.CreatedAt, err = time.Parse(types.SfDateTimeLayout, val)
if err != nil {
@ -84,7 +85,7 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace)
}
break
case "LastModifiedDate":
case "lastmodifieddate":
if val != "" {
tt, err := time.Parse(types.SfDateTimeLayout, val)
u.UpdatedAt = &tt
@ -95,7 +96,7 @@ func importUsers(ctx context.Context, is *types.ImportSource, ns *cct.Namespace)
break
// ignore deleted values, as SF provides minimal info about those
case "IsDeleted":
case "isdeleted":
if val == "1" {
goto looper
}

36
pkg/ngimporter/main.go
View File

@ -9,7 +9,6 @@ import (
"sync"
"github.com/cortezaproject/corteza-server/compose/repository"
"github.com/cortezaproject/corteza-server/compose/service"
cct "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/ngimporter/types"
"github.com/schollz/progressbar/v2"
@ -37,7 +36,8 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er
// contains warnings raised by the pre process steps
var preProcW []string
imp := &Importer{}
svcMod := service.DefaultModule.With(ctx)
db := repository.DB(ctx)
modRepo := repository.Module(ctx, db)
var err error
// import users
@ -87,7 +87,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er
for _, nIs := range nIss {
// preload module
mod, err := svcMod.FindByHandle(ns.ID, nIs.Name)
mod, err := findModuleByHandle(modRepo, ns.ID, nIs.Name)
if err != nil {
preProcW = append(preProcW, err.Error()+" "+nIs.Name)
continue
@ -142,7 +142,7 @@ func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) er
continue
}
mm, err := svcMod.FindByID(ns.ID, vv)
mm, err := findModuleByID(modRepo, ns.ID, vv)
if err != nil {
preProcW = append(preProcW, err.Error()+" "+nIs.Name+" "+f.Name+" "+modID)
continue
@ -274,3 +274,31 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error {
return nil
})
}
func findModuleByID(repo repository.ModuleRepository, namespaceID, moduleID uint64) (*cct.Module, error) {
var err error
mod, err := repo.FindByID(namespaceID, moduleID)
if err != nil {
return nil, err
}
mod.Fields, err = repo.FindFields(mod.ID)
if err != nil {
return nil, err
}
return mod, nil
}
func findModuleByHandle(repo repository.ModuleRepository, namespaceID uint64, handle string) (*cct.Module, error) {
var err error
mod, err := repo.FindByHandle(namespaceID, handle)
if err != nil {
return nil, err
}
mod.Fields, err = repo.FindFields(mod.ID)
if err != nil {
return nil, err
}
return mod, nil
}

5
pkg/ngimporter/map_data.go
View File

@ -128,7 +128,10 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) {
}
}
val := record[hMap[from]]
val := ""
if i, has := hMap[from]; has {
val = record[i]
}
// handle data join
if strings.Contains(from, ".") {

20
pkg/ngimporter/types/eval.go
View File

@ -11,9 +11,23 @@ import (
// Glang generates a gval language, that can be used for expression evaluation
func GLang() gval.Language {
return gval.NewLanguage(
gval.JSON(),
gval.Arithmetic(),
gval.PropositionalLogic(),
gval.Full(),
gval.Function("f64", func(v string) (float64, error) {
nn, err := strconv.ParseFloat(v, 64)
if err != nil {
return 0, err
}
return nn, nil
}),
gval.Function("concat", func(vv ...string) (string, error) {
out := ""
for _, v := range vv {
out += v
}
return out, nil
}),
gval.Function("numFmt", func(number, format string) (string, error) {
nn, err := strconv.ParseFloat(number, 64)

7
pkg/ngimporter/types/general.go
View File

@ -1,10 +1,13 @@
package types
import "unicode/utf8"
import (
"time"
"unicode/utf8"
)
const (
// SfDateTimeLayout represents the date-time template used by sales force
SfDateTimeLayout = "2006-01-02 15:04:05"
SfDateTimeLayout = time.RFC3339
// DateOnlyLayout represents our internal date only date-time fields
DateOnlyLayout = "2006-01-02"
// TimeOnlyLayout represents our internal time only date-time fields

66
pkg/ngimporter/types/import_node.go
View File

@ -6,12 +6,13 @@ import (
"errors"
"fmt"
"io"
"strconv"
"log"
"strings"
"sync"
"time"
"github.com/cortezaproject/corteza-server/compose/repository"
cv "github.com/cortezaproject/corteza-server/compose/service/values"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/schollz/progressbar/v2"
)
@ -318,6 +319,8 @@ func isSysField(f string) bool {
// updates the given node's record values that depend on another record
func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error {
s := cv.Sanitizer()
for _, r := range n.records {
for _, v := range r.Values {
var f *types.ModuleField
@ -346,6 +349,7 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error {
if mod, ok := n.idMap[ref]; ok {
if vv, ok := mod[val]; ok {
v.Value = vv
v.Updated = true
} else {
v.Value = ""
continue
@ -365,10 +369,13 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error {
}
}
nv = s.Run(n.Module, nv)
r.Values = nv
err := repo.UpdateValues(r.ID, r.Values)
if err != nil {
return err
log.Printf("[issue] db.UpdateValues | %d | %s | %s \n", r.ID, r.Values.String(), err.Error())
// return err
}
}
return nil
@ -377,6 +384,7 @@ func (n *ImportNode) correctRecordRefs(repo repository.RecordRepository) error {
// imports the given node's source
func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.RecordRepository) (Map, error) {
mapping := make(Map)
s := cv.Sanitizer()
for {
looper:
@ -389,6 +397,10 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
return nil, err
}
if record[0] == "" {
continue
}
rr := &types.Record{
ModuleID: n.Module.ID,
NamespaceID: n.Namespace.ID,
@ -411,19 +423,19 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
// system values should be kept on the record's root level
if isSysField(h) {
switch h {
case "OwnerId":
switch strings.ToLower(h) {
case "ownerid":
rr.OwnedBy = users[val]
break
// ignore deleted values, as SF provides minimal info about those
case "IsDeleted":
if val == "1" {
case "isdeleted":
if val == "1" || strings.ToLower(val) == "true" {
goto looper
}
break
case "CreatedDate":
case "createddate":
if val != "" {
rr.CreatedAt, err = time.Parse(SfDateTimeLayout, val)
if err != nil {
@ -432,15 +444,15 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
}
break
case "CreatedById":
case "createdbyid":
rr.CreatedBy = users[val]
break
case "LastModifiedById":
case "lastmodifiedbyid":
rr.UpdatedBy = users[val]
break
case "LastModifiedDate":
case "lastmodifieddate":
if val != "" {
tt, err := time.Parse(SfDateTimeLayout, val)
rr.UpdatedAt = &tt
@ -520,13 +532,6 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
continue
}
if f.Kind == "DateTime" {
val, err = assureDateFormat(val, f.Options)
if err != nil {
return nil, err
}
}
values = append(values, val)
}
}
@ -538,18 +543,19 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
return nil, err
}
rv := &types.RecordValue{
Name: h,
Value: v,
Place: uint(i),
Name: h,
Value: v,
Place: uint(i),
Updated: true,
}
if f.IsRef() && v != "" {
rv.Ref, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, err
}
// ref values of spliced nodes should get updated later
if n.isSpliced && f.IsRef() {
rv.Updated = false
}
recordValues = append(recordValues, rv)
}
recordValues = s.Run(n.Module, recordValues)
}
}
@ -563,9 +569,13 @@ func (n *ImportNode) importNodeSource(users map[string]uint64, repo repository.R
for _, v := range recordValues {
v.RecordID = r.ID
}
err = repo.UpdateValues(r.ID, recordValues)
if err != nil {
return nil, err
if !n.isSpliced {
err = repo.UpdateValues(r.ID, recordValues)
if err != nil {
log.Printf("[issue] db.UpdateValues | %d | %s | %s \n", r.ID, recordValues.String(), err.Error())
// return nil, err
}
}
// spliced nodes should preserve their records for later ref processing