diff --git a/compose/app.go b/compose/app.go index 69945ce34..bc22a5c53 100644 --- a/compose/app.go +++ b/compose/app.go @@ -102,7 +102,7 @@ func (app *App) RegisterCliCommands(p *cobra.Command) { p.AddCommand( commands.Importer(), commands.Exporter(), - commands.ImporterNG(), + commands.NGImporter(), // temp command, will be removed in 2020.6 automation.ScriptExporter(SERVICE), ) diff --git a/compose/commands/migrator.go b/compose/commands/ng_importer.go similarity index 68% rename from compose/commands/migrator.go rename to compose/commands/ng_importer.go index ec4e85965..14887b9fc 100644 --- a/compose/commands/migrator.go +++ b/compose/commands/ng_importer.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" - "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -19,13 +18,13 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/cli" - ngi "github.com/cortezaproject/corteza-server/pkg/ngImporter" - ngt "github.com/cortezaproject/corteza-server/pkg/ngImporter/types" + ngi "github.com/cortezaproject/corteza-server/pkg/ngimporter" + ngt "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" ) -func ImporterNG() *cobra.Command { +func NGImporter() *cobra.Command { cmd := &cobra.Command{ - Use: "ng-import", + Use: "ng-importer", Short: "Importer next-gen", Run: func(cmd *cobra.Command, args []string) { @@ -37,7 +36,7 @@ func ImporterNG() *cobra.Command { ns *types.Namespace err error - mg []ngt.ImportSource + iss []ngt.ImportSource ) svcNs := service.DefaultNamespace.With(ctx) @@ -70,12 +69,12 @@ func ImporterNG() *cobra.Command { ext := filepath.Ext(info.Name()) name := info.Name()[0 : len(info.Name())-len(ext)] - mm := importSource(mg, name) + mm := importSource(iss, name) mm.Name = name mm.Path = path mm.Source = file - mg = addImportSource(mg, mm) + iss = addImportSource(iss, mm) } return nil }) @@ -83,44 +82,41 @@ func ImporterNG() *cobra.Command { // load meta files if metaFlag != "" { err = filepath.Walk(metaFlag, func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(info.Name(), ".map.json") { + if strings.HasSuffix(info.Name(), ngt.MetaMapExt) { file, err := os.Open(path) if err != nil { log.Fatal(err) } - ext := filepath.Ext(info.Name()) // @todo improve this!! - name := info.Name()[0 : len(info.Name())-len(ext)-4] - mm := importSource(mg, name) + name := info.Name()[0 : len(info.Name())-len(ngt.MetaMapExt)] + mm := importSource(iss, name) mm.Name = name mm.DataMap = file - mg = addImportSource(mg, mm) - } else if strings.HasSuffix(info.Name(), ".join.json") { + iss = addImportSource(iss, mm) + } else if strings.HasSuffix(info.Name(), ngt.MetaJoinExt) { file, err := os.Open(path) if err != nil { log.Fatal(err) } - ext := filepath.Ext(info.Name()) // @todo improve this!! - name := info.Name()[0 : len(info.Name())-len(ext)-5] - mm := importSource(mg, name) + name := info.Name()[0 : len(info.Name())-len(ngt.MetaJoinExt)] + mm := importSource(iss, name) mm.Name = name mm.SourceJoin = file - mg = addImportSource(mg, mm) - } else if strings.HasSuffix(info.Name(), ".value.json") { + iss = addImportSource(iss, mm) + } else if strings.HasSuffix(info.Name(), ngt.MetaValueExt) { file, err := os.Open(path) if err != nil { log.Fatal(err) } - ext := filepath.Ext(info.Name()) // @todo improve this!! - name := info.Name()[0 : len(info.Name())-len(ext)-6] - mm := importSource(mg, name) + name := info.Name()[0 : len(info.Name())-len(ngt.MetaValueExt)] + mm := importSource(iss, name) mm.Name = name var vmp map[string]map[string]string @@ -131,25 +127,25 @@ func ImporterNG() *cobra.Command { } mm.ValueMap = vmp - mg = addImportSource(mg, mm) + iss = addImportSource(iss, mm) } return nil }) if err != nil { - panic(err) + log.Fatal(err) } } - // clean up migrateables + // remove nodes with no import source hasW := false out := make([]ngt.ImportSource, 0) - for _, m := range mg { - if m.Source != nil { - out = append(out, m) + for _, is := range iss { + if is.Source != nil { + out = append(out, is) } else { hasW = true - spew.Dump("[warning] migrationNode.missingSource " + m.Name) + log.Println("[warning] ImportSOurce.missingSource " + is.Name) } } @@ -157,15 +153,14 @@ func ImporterNG() *cobra.Command { var rsp string fmt.Print("warnings detected; continue [y/N]? ") _, err := fmt.Scanln(&rsp) - rsp = strings.ToLower(rsp) if err != nil || rsp != "y" && rsp != "yes" { - log.Fatal("migration aborted due to warnings") + log.Fatal("import aborted due to warnings") } } - err = ngi.Import(out, ns, ctx) + err = ngi.Import(ctx, out, ns) if err != nil { panic(err) } @@ -182,22 +177,22 @@ func ImporterNG() *cobra.Command { // small helper functions for import source node management -func importSource(mg []ngt.ImportSource, name string) ngt.ImportSource { - for _, m := range mg { - if m.Name == name { - return m +func importSource(iss []ngt.ImportSource, name string) ngt.ImportSource { + for _, is := range iss { + if is.Name == name { + return is } } return ngt.ImportSource{} } -func addImportSource(mg []ngt.ImportSource, mm ngt.ImportSource) []ngt.ImportSource { - for i, m := range mg { - if m.Name == mm.Name { - mg[i] = mm - return mg +func addImportSource(iss []ngt.ImportSource, nis ngt.ImportSource) []ngt.ImportSource { + for i, is := range iss { + if is.Name == nis.Name { + iss[i] = nis + return iss } } - return append(mg, mm) + return append(iss, nis) } diff --git a/pkg/ngImporter/README.adoc b/pkg/ngimporter/README.adoc similarity index 100% rename from pkg/ngImporter/README.adoc rename to pkg/ngimporter/README.adoc diff --git a/pkg/ngImporter/importUsers.go b/pkg/ngimporter/import_users.go similarity index 96% rename from pkg/ngImporter/importUsers.go rename to pkg/ngimporter/import_users.go index c41f67f33..3f9ac3e20 100644 --- a/pkg/ngImporter/importUsers.go +++ b/pkg/ngimporter/import_users.go @@ -1,4 +1,4 @@ -package migrate +package ngimporter import ( "bytes" @@ -9,7 +9,7 @@ import ( "github.com/cortezaproject/corteza-server/compose/repository" cct "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/ngImporter/types" + "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" sysRepo "github.com/cortezaproject/corteza-server/system/repository" sysTypes "github.com/cortezaproject/corteza-server/system/types" ) diff --git a/pkg/ngImporter/joinData.go b/pkg/ngimporter/join_data.go similarity index 96% rename from pkg/ngImporter/joinData.go rename to pkg/ngimporter/join_data.go index f20e0e53b..bff92c37a 100644 --- a/pkg/ngImporter/joinData.go +++ b/pkg/ngimporter/join_data.go @@ -1,4 +1,4 @@ -package migrate +package ngimporter import ( "encoding/csv" @@ -9,7 +9,7 @@ import ( "regexp" "strings" - "github.com/cortezaproject/corteza-server/pkg/ngImporter/types" + "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" ) type ( @@ -86,10 +86,10 @@ func joinData(iss []types.ImportSource) ([]types.ImportSource, error) { } nd.aliasMap[expr.baseFieldAlias] = expr.baseFields - for _, m := range iss { - if m.Name == expr.joinModule { + for _, is := range iss { + if is.Name == expr.joinModule { if _, ok := joinedNodes[expr.joinModule]; !ok { - ww := m + ww := is joinedNodes[expr.joinModule] = &types.JoinNode{ Mg: &ww, Name: ww.Name, @@ -115,7 +115,6 @@ func joinData(iss []types.ImportSource) ([]types.ImportSource, error) { jn.Records = make([]map[string]string, 0) reader := csv.NewReader(jn.Mg.Source) - // header header, err := reader.Read() if err == io.EOF { break @@ -201,7 +200,7 @@ func splitExpr(base, joined string) joinEval { rr.baseFields = strings.Split(mx[1], ",") rr.baseFieldAlias = mx[2] - // joined node + // join node rx = regexp.MustCompile(`(?P\w+)\.\[?(?P[\w,]+)\]?`) mx = rx.FindStringSubmatch(joined) rr.joinModule = mx[1] diff --git a/pkg/ngImporter/main.go b/pkg/ngimporter/main.go similarity index 85% rename from pkg/ngImporter/main.go rename to pkg/ngimporter/main.go index a0d862f0c..8212befed 100644 --- a/pkg/ngImporter/main.go +++ b/pkg/ngimporter/main.go @@ -1,18 +1,17 @@ -package migrate +package ngimporter import ( "context" "encoding/csv" - "fmt" "io" + "log" "strconv" "sync" "github.com/cortezaproject/corteza-server/compose/repository" "github.com/cortezaproject/corteza-server/compose/service" cct "github.com/cortezaproject/corteza-server/compose/types" - "github.com/cortezaproject/corteza-server/pkg/ngImporter/types" - "github.com/davecgh/go-spew/spew" + "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" "github.com/schollz/progressbar/v2" ) @@ -34,7 +33,7 @@ type ( // * build graph from ImportNodes based on the provided ImportSource nodes // * remove cycles from the given graph // * import data based on node dependencies -func Import(iss []types.ImportSource, ns *cct.Namespace, ctx context.Context) error { +func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) error { // contains warnings raised by the pre process steps var preProcW []string imp := &Importer{} @@ -163,7 +162,10 @@ func Import(iss []types.ImportSource, ns *cct.Namespace, ctx context.Context) er } } - spew.Dump("PRE-PROC WARNINGS", preProcW) + log.Println("PRE-PROC WARNINGS") + for _, w := range preProcW { + log.Printf("[warning] %s\n", w) + } imp.RemoveCycles() @@ -174,23 +176,26 @@ func Import(iss []types.ImportSource, ns *cct.Namespace, ctx context.Context) er } } - fmt.Printf("import.prepared\n") - fmt.Printf("no. of nodes %d\n", len(imp.nodes)) - fmt.Printf("no. of entry points %d\n", len(imp.Leafs)) + log.Printf("[importer] prepared\n") + log.Printf("[importer] node count: %d\n", len(imp.nodes)) + log.Printf("[importer] leaf count: %d\n", len(imp.Leafs)) + log.Println("[importer] started") err = imp.Import(ctx, uMap) if err != nil { + log.Println("[importer] failed") return err } + log.Println("[importer] finished") return nil } // AddNode attempts to add the given node into the graph. If the node can already be // identified, the two nodes are merged. -func (m *Importer) AddNode(n *types.ImportNode) *types.ImportNode { +func (imp *Importer) AddNode(n *types.ImportNode) *types.ImportNode { var fn *types.ImportNode - for _, nn := range m.nodes { + for _, nn := range imp.nodes { if nn.CompareTo(n) { fn = nn break @@ -198,7 +203,7 @@ func (m *Importer) AddNode(n *types.ImportNode) *types.ImportNode { } if fn == nil { - m.nodes = append(m.nodes, n) + imp.nodes = append(imp.nodes, n) return n } @@ -208,13 +213,13 @@ func (m *Importer) AddNode(n *types.ImportNode) *types.ImportNode { // RemoveCycles removes all cycles in the given graph, by restructuring/recreating the nodes // and their dependencies. -func (m *Importer) RemoveCycles() { +func (imp *Importer) RemoveCycles() { splice := func(n *types.ImportNode, from *types.ImportNode) { spl := n.Splice(from) - m.AddNode(spl) + imp.AddNode(spl) } - for _, n := range m.nodes { + for _, n := range imp.nodes { if !n.Visited { n.SeekCycles(splice) } @@ -223,11 +228,6 @@ func (m *Importer) RemoveCycles() { // Import runs the import over each ImportNode in the given graph func (m *Importer) Import(ctx context.Context, users map[string]uint64) error { - fmt.Println("(•_•)") - fmt.Println("(-•_•)>⌐■-■") - fmt.Println("(⌐■_■)") - fmt.Print("\n\n\n") - db := repository.DB(ctx) repoRecord := repository.Record(ctx, db) bar := progressbar.New(len(m.nodes)) @@ -248,7 +248,7 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error { for len(ch) > 0 { pp := <-ch if pp.Err != nil { - spew.Dump("ERR", pp.Err, pp.Node.Stringify()) + log.Printf("[importer] node %s failed with %s\n", pp.Err.Error(), pp.Node.Stringify()) return pp.Err } @@ -271,11 +271,6 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error { m.Leafs = nl } - fmt.Print("\n\n\n") - fmt.Println("(⌐■_■)") - fmt.Println("(-•_•)>⌐■-■") - fmt.Println("(•_•)") - return nil }) } diff --git a/pkg/ngImporter/mapData.go b/pkg/ngimporter/map_data.go similarity index 90% rename from pkg/ngImporter/mapData.go rename to pkg/ngimporter/map_data.go index 61f39f807..ce6014166 100644 --- a/pkg/ngImporter/mapData.go +++ b/pkg/ngimporter/map_data.go @@ -1,4 +1,4 @@ -package migrate +package ngimporter import ( "bytes" @@ -10,7 +10,7 @@ import ( "io/ioutil" "strings" - "github.com/cortezaproject/corteza-server/pkg/ngImporter/types" + "github.com/cortezaproject/corteza-server/pkg/ngimporter/types" ) type ( @@ -89,7 +89,7 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) { // find applicable maps, that can be used for the given row. // the system allows composition, so all applicable maps are used. for _, strmp := range dataMap { - if checkWhere(strmp["where"], record, hMap) { + if ok, err := checkWhere(strmp["where"], record, hMap); ok && err == nil { maps, ok := strmp["map"].([]interface{}) if !ok { return nil, errors.New("dataMap.invalidMap " + is.Name) @@ -153,6 +153,8 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) { bufs[nm].header = append(bufs[nm].header, nmF) } } + } else if err != nil { + return nil, err } } @@ -180,21 +182,20 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) { return rr, nil } -// quick and dirty function to check the map's where condition. -// improve with our QL package -func checkWhere(where interface{}, row []string, hMap map[string]int) bool { +// checks if the given condition passes for the given row +func checkWhere(where interface{}, row []string, hMap map[string]int) (bool, error) { if where == nil { - return true + return true, nil } - ww, ok := where.(string) + wh, ok := where.(string) if !ok { - return true + return true, nil } - ev, err := types.GLang().NewEvaluable(ww) + ev, err := types.ExprLang.NewEvaluable(wh) if err != nil { - panic(err) + return false, err } // prep payload @@ -205,8 +206,8 @@ func checkWhere(where interface{}, row []string, hMap map[string]int) bool { rr, err := ev.EvalBool(context.Background(), pr) if err != nil { - panic(err) + return false, err } - return rr + return rr, nil } diff --git a/pkg/ngImporter/types/eval.go b/pkg/ngimporter/types/eval.go similarity index 100% rename from pkg/ngImporter/types/eval.go rename to pkg/ngimporter/types/eval.go diff --git a/pkg/ngImporter/types/general.go b/pkg/ngimporter/types/general.go similarity index 92% rename from pkg/ngImporter/types/general.go rename to pkg/ngimporter/types/general.go index 5bfa60797..4a871f161 100644 --- a/pkg/ngImporter/types/general.go +++ b/pkg/ngimporter/types/general.go @@ -14,6 +14,10 @@ const ( EvalPrefix = "=EVL=" UserModHandle = "User" + + MetaMapExt = ".map.json" + MetaJoinExt = ".join.json" + MetaValueExt = ".value.json" ) var ( diff --git a/pkg/ngImporter/types/importNode.go b/pkg/ngimporter/types/import_node.go similarity index 97% rename from pkg/ngImporter/types/importNode.go rename to pkg/ngimporter/types/import_node.go index 504fb6cb1..568a57af5 100644 --- a/pkg/ngImporter/types/importNode.go +++ b/pkg/ngimporter/types/import_node.go @@ -141,7 +141,7 @@ func (n *ImportNode) Import(repoRecord repository.RecordRepository, users map[st for _, p := range pps { rtr = append(rtr, p) - // pass mapping object to the node's parend so it can migrate it's data + // pass mapping object to the node's parent so it can map handle dependency refs p.addMap(fmt.Sprint(n.Module.ID), mapping) p.LinkRemove(n) } @@ -236,7 +236,7 @@ func (n *ImportNode) removeIfPresent(rem *ImportNode, list []*ImportNode) []*Imp return list } -// traverses the graph and notifies us of any cycles +// SeekCycles finds cycles & calls the given function func (n *ImportNode) SeekCycles(cycle func(n *ImportNode, to *ImportNode)) { n.inPath = true n.Visited = true @@ -247,9 +247,6 @@ func (n *ImportNode) SeekCycles(cycle func(n *ImportNode, to *ImportNode)) { } for _, nn := range cc { - if n.Name == "client" { - } - if nn.inPath { cycle(n, nn) } else { @@ -260,18 +257,6 @@ func (n *ImportNode) SeekCycles(cycle func(n *ImportNode, to *ImportNode)) { n.inPath = false } -func (n *ImportNode) DFS() { - n.inPath = true - - for _, nn := range n.Children { - if !nn.inPath { - nn.DFS() - } - } - - n.inPath = false -} - // clones the given node func (n *ImportNode) clone() *ImportNode { return &ImportNode{ @@ -288,6 +273,8 @@ func (n *ImportNode) clone() *ImportNode { Namespace: n.Namespace, Reader: n.Reader, Header: n.Header, + FieldMap: n.FieldMap, + ValueMap: n.ValueMap, } } diff --git a/pkg/ngImporter/types/ImportSource.go b/pkg/ngimporter/types/import_source.go similarity index 100% rename from pkg/ngImporter/types/ImportSource.go rename to pkg/ngimporter/types/import_source.go