DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Post feedback tweaks

Browse Source
This commit is contained in:
Tomaž Jerman
2020-03-25 13:57:36 +01:00
parent ecf85b340d
commit e6ab67d2fb
11 changed files with 89 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/app.go
+1 -1
View File
@@ -102,7 +102,7 @@ func (app *App) RegisterCliCommands(p *cobra.Command) {
p.AddCommand(
commands.Importer(),
commands.Exporter(),
commands.ImporterNG(),
commands.NGImporter(),
// temp command, will be removed in 2020.6
automation.ScriptExporter(SERVICE),
)
compose/commands/migrator.go → compose/commands/ng_importer.go
+37 -42
View File
@@ -10,7 +10,6 @@ import (
"strconv"
"strings"
"github.com/davecgh/go-spew/spew"
"github.com/pkg/errors"
"github.com/spf13/cobra"
@@ -19,13 +18,13 @@ import (
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/cli"
ngi "github.com/cortezaproject/corteza-server/pkg/ngImporter"
ngt "github.com/cortezaproject/corteza-server/pkg/ngImporter/types"
ngi "github.com/cortezaproject/corteza-server/pkg/ngimporter"
ngt "github.com/cortezaproject/corteza-server/pkg/ngimporter/types"
)
func ImporterNG() *cobra.Command {
func NGImporter() *cobra.Command {
cmd := &cobra.Command{
Use: "ng-import",
Use: "ng-importer",
Short: "Importer next-gen",
Run: func(cmd *cobra.Command, args []string) {
@@ -37,7 +36,7 @@ func ImporterNG() *cobra.Command {
ns *types.Namespace
err error
mg []ngt.ImportSource
iss []ngt.ImportSource
)
svcNs := service.DefaultNamespace.With(ctx)
@@ -70,12 +69,12 @@ func ImporterNG() *cobra.Command {
ext := filepath.Ext(info.Name())
name := info.Name()[0 : len(info.Name())-len(ext)]
mm := importSource(mg, name)
mm := importSource(iss, name)
mm.Name = name
mm.Path = path
mm.Source = file
mg = addImportSource(mg, mm)
iss = addImportSource(iss, mm)
}
return nil
})
@@ -83,44 +82,41 @@ func ImporterNG() *cobra.Command {
// load meta files
if metaFlag != "" {
err = filepath.Walk(metaFlag, func(path string, info os.FileInfo, err error) error {
if strings.HasSuffix(info.Name(), ".map.json") {
if strings.HasSuffix(info.Name(), ngt.MetaMapExt) {
file, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
ext := filepath.Ext(info.Name())
// @todo improve this!!
name := info.Name()[0 : len(info.Name())-len(ext)-4]
mm := importSource(mg, name)
name := info.Name()[0 : len(info.Name())-len(ngt.MetaMapExt)]
mm := importSource(iss, name)
mm.Name = name
mm.DataMap = file
mg = addImportSource(mg, mm)
} else if strings.HasSuffix(info.Name(), ".join.json") {
iss = addImportSource(iss, mm)
} else if strings.HasSuffix(info.Name(), ngt.MetaJoinExt) {
file, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
ext := filepath.Ext(info.Name())
// @todo improve this!!
name := info.Name()[0 : len(info.Name())-len(ext)-5]
mm := importSource(mg, name)
name := info.Name()[0 : len(info.Name())-len(ngt.MetaJoinExt)]
mm := importSource(iss, name)
mm.Name = name
mm.SourceJoin = file
mg = addImportSource(mg, mm)
} else if strings.HasSuffix(info.Name(), ".value.json") {
iss = addImportSource(iss, mm)
} else if strings.HasSuffix(info.Name(), ngt.MetaValueExt) {
file, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
ext := filepath.Ext(info.Name())
// @todo improve this!!
name := info.Name()[0 : len(info.Name())-len(ext)-6]
mm := importSource(mg, name)
name := info.Name()[0 : len(info.Name())-len(ngt.MetaValueExt)]
mm := importSource(iss, name)
mm.Name = name
var vmp map[string]map[string]string
@@ -131,25 +127,25 @@ func ImporterNG() *cobra.Command {
}
mm.ValueMap = vmp
mg = addImportSource(mg, mm)
iss = addImportSource(iss, mm)
}
return nil
})
if err != nil {
panic(err)
log.Fatal(err)
}
}
// clean up migrateables
// remove nodes with no import source
hasW := false
out := make([]ngt.ImportSource, 0)
for _, m := range mg {
if m.Source != nil {
out = append(out, m)
for _, is := range iss {
if is.Source != nil {
out = append(out, is)
} else {
hasW = true
spew.Dump("[warning] migrationNode.missingSource " + m.Name)
log.Println("[warning] ImportSOurce.missingSource " + is.Name)
}
}
@@ -157,15 +153,14 @@ func ImporterNG() *cobra.Command {
var rsp string
fmt.Print("warnings detected; continue [y/N]? ")
_, err := fmt.Scanln(&rsp)
rsp = strings.ToLower(rsp)
if err != nil || rsp != "y" && rsp != "yes" {
log.Fatal("migration aborted due to warnings")
log.Fatal("import aborted due to warnings")
}
}
err = ngi.Import(out, ns, ctx)
err = ngi.Import(ctx, out, ns)
if err != nil {
panic(err)
}
@@ -182,22 +177,22 @@ func ImporterNG() *cobra.Command {
// small helper functions for import source node management
func importSource(mg []ngt.ImportSource, name string) ngt.ImportSource {
for _, m := range mg {
if m.Name == name {
return m
func importSource(iss []ngt.ImportSource, name string) ngt.ImportSource {
for _, is := range iss {
if is.Name == name {
return is
}
}
return ngt.ImportSource{}
}
func addImportSource(mg []ngt.ImportSource, mm ngt.ImportSource) []ngt.ImportSource {
for i, m := range mg {
if m.Name == mm.Name {
mg[i] = mm
return mg
func addImportSource(iss []ngt.ImportSource, nis ngt.ImportSource) []ngt.ImportSource {
for i, is := range iss {
if is.Name == nis.Name {
iss[i] = nis
return iss
}
}
return append(mg, mm)
return append(iss, nis)
}
pkg/ngImporter/README.adoc → pkg/ngimporter/README.adoc
View File
pkg/ngImporter/importUsers.go → pkg/ngimporter/import_users.go
+2 -2
View File
@@ -1,4 +1,4 @@
package migrate
package ngimporter
import (
"bytes"
@@ -9,7 +9,7 @@ import (
"github.com/cortezaproject/corteza-server/compose/repository"
cct "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/ngImporter/types"
"github.com/cortezaproject/corteza-server/pkg/ngimporter/types"
sysRepo "github.com/cortezaproject/corteza-server/system/repository"
sysTypes "github.com/cortezaproject/corteza-server/system/types"
)
pkg/ngImporter/joinData.go → pkg/ngimporter/join_data.go
+6 -7
View File
@@ -1,4 +1,4 @@
package migrate
package ngimporter
import (
"encoding/csv"
@@ -9,7 +9,7 @@ import (
"regexp"
"strings"
"github.com/cortezaproject/corteza-server/pkg/ngImporter/types"
"github.com/cortezaproject/corteza-server/pkg/ngimporter/types"
)
type (
@@ -86,10 +86,10 @@ func joinData(iss []types.ImportSource) ([]types.ImportSource, error) {
}
nd.aliasMap[expr.baseFieldAlias] = expr.baseFields
for _, m := range iss {
if m.Name == expr.joinModule {
for _, is := range iss {
if is.Name == expr.joinModule {
if _, ok := joinedNodes[expr.joinModule]; !ok {
ww := m
ww := is
joinedNodes[expr.joinModule] = &types.JoinNode{
Mg: &ww,
Name: ww.Name,
@@ -115,7 +115,6 @@ func joinData(iss []types.ImportSource) ([]types.ImportSource, error) {
jn.Records = make([]map[string]string, 0)
reader := csv.NewReader(jn.Mg.Source)
// header
header, err := reader.Read()
if err == io.EOF {
break
@@ -201,7 +200,7 @@ func splitExpr(base, joined string) joinEval {
rr.baseFields = strings.Split(mx[1], ",")
rr.baseFieldAlias = mx[2]
// joined node
// join node
rx = regexp.MustCompile(`(?P<jm>\w+)\.\[?(?P<jmf>[\w,]+)\]?`)
mx = rx.FindStringSubmatch(joined)
rr.joinModule = mx[1]
pkg/ngImporter/main.go → pkg/ngimporter/main.go
+21 -26
View File
@@ -1,18 +1,17 @@
package migrate
package ngimporter
import (
"context"
"encoding/csv"
"fmt"
"io"
"log"
"strconv"
"sync"
"github.com/cortezaproject/corteza-server/compose/repository"
"github.com/cortezaproject/corteza-server/compose/service"
cct "github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/ngImporter/types"
"github.com/davecgh/go-spew/spew"
"github.com/cortezaproject/corteza-server/pkg/ngimporter/types"
"github.com/schollz/progressbar/v2"
)
@@ -34,7 +33,7 @@ type (
// * build graph from ImportNodes based on the provided ImportSource nodes
// * remove cycles from the given graph
// * import data based on node dependencies
func Import(iss []types.ImportSource, ns *cct.Namespace, ctx context.Context) error {
func Import(ctx context.Context, iss []types.ImportSource, ns *cct.Namespace) error {
// contains warnings raised by the pre process steps
var preProcW []string
imp := &Importer{}
@@ -163,7 +162,10 @@ func Import(iss []types.ImportSource, ns *cct.Namespace, ctx context.Context) er
}
}
spew.Dump("PRE-PROC WARNINGS", preProcW)
log.Println("PRE-PROC WARNINGS")
for _, w := range preProcW {
log.Printf("[warning] %s\n", w)
}
imp.RemoveCycles()
@@ -174,23 +176,26 @@ func Import(iss []types.ImportSource, ns *cct.Namespace, ctx context.Context) er
}
}
fmt.Printf("import.prepared\n")
fmt.Printf("no. of nodes %d\n", len(imp.nodes))
fmt.Printf("no. of entry points %d\n", len(imp.Leafs))
log.Printf("[importer] prepared\n")
log.Printf("[importer] node count: %d\n", len(imp.nodes))
log.Printf("[importer] leaf count: %d\n", len(imp.Leafs))
log.Println("[importer] started")
err = imp.Import(ctx, uMap)
if err != nil {
log.Println("[importer] failed")
return err
}
log.Println("[importer] finished")
return nil
}
// AddNode attempts to add the given node into the graph. If the node can already be
// identified, the two nodes are merged.
func (m *Importer) AddNode(n *types.ImportNode) *types.ImportNode {
func (imp *Importer) AddNode(n *types.ImportNode) *types.ImportNode {
var fn *types.ImportNode
for _, nn := range m.nodes {
for _, nn := range imp.nodes {
if nn.CompareTo(n) {
fn = nn
break
@@ -198,7 +203,7 @@ func (m *Importer) AddNode(n *types.ImportNode) *types.ImportNode {
}
if fn == nil {
m.nodes = append(m.nodes, n)
imp.nodes = append(imp.nodes, n)
return n
}
@@ -208,13 +213,13 @@ func (m *Importer) AddNode(n *types.ImportNode) *types.ImportNode {
// RemoveCycles removes all cycles in the given graph, by restructuring/recreating the nodes
// and their dependencies.
func (m *Importer) RemoveCycles() {
func (imp *Importer) RemoveCycles() {
splice := func(n *types.ImportNode, from *types.ImportNode) {
spl := n.Splice(from)
m.AddNode(spl)
imp.AddNode(spl)
}
for _, n := range m.nodes {
for _, n := range imp.nodes {
if !n.Visited {
n.SeekCycles(splice)
}
@@ -223,11 +228,6 @@ func (m *Importer) RemoveCycles() {
// Import runs the import over each ImportNode in the given graph
func (m *Importer) Import(ctx context.Context, users map[string]uint64) error {
fmt.Println("(•_•)")
fmt.Println("(-•_•)>⌐■-■")
fmt.Println("(⌐■_■)")
fmt.Print("\n\n\n")
db := repository.DB(ctx)
repoRecord := repository.Record(ctx, db)
bar := progressbar.New(len(m.nodes))
@@ -248,7 +248,7 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error {
for len(ch) > 0 {
pp := <-ch
if pp.Err != nil {
spew.Dump("ERR", pp.Err, pp.Node.Stringify())
log.Printf("[importer] node %s failed with %s\n", pp.Err.Error(), pp.Node.Stringify())
return pp.Err
}
@@ -271,11 +271,6 @@ func (m *Importer) Import(ctx context.Context, users map[string]uint64) error {
m.Leafs = nl
}
fmt.Print("\n\n\n")
fmt.Println("(⌐■_■)")
fmt.Println("(-•_•)>⌐■-■")
fmt.Println("(•_•)")
return nil
})
}
pkg/ngImporter/mapData.go → pkg/ngimporter/map_data.go
+14 -13
View File
@@ -1,4 +1,4 @@
package migrate
package ngimporter
import (
"bytes"
@@ -10,7 +10,7 @@ import (
"io/ioutil"
"strings"
"github.com/cortezaproject/corteza-server/pkg/ngImporter/types"
"github.com/cortezaproject/corteza-server/pkg/ngimporter/types"
)
type (
@@ -89,7 +89,7 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) {
// find applicable maps, that can be used for the given row.
// the system allows composition, so all applicable maps are used.
for _, strmp := range dataMap {
if checkWhere(strmp["where"], record, hMap) {
if ok, err := checkWhere(strmp["where"], record, hMap); ok && err == nil {
maps, ok := strmp["map"].([]interface{})
if !ok {
return nil, errors.New("dataMap.invalidMap " + is.Name)
@@ -153,6 +153,8 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) {
bufs[nm].header = append(bufs[nm].header, nmF)
}
}
} else if err != nil {
return nil, err
}
}
@@ -180,21 +182,20 @@ func mapData(is types.ImportSource) ([]types.ImportSource, error) {
return rr, nil
}
// quick and dirty function to check the map's where condition.
// improve with our QL package
func checkWhere(where interface{}, row []string, hMap map[string]int) bool {
// checks if the given condition passes for the given row
func checkWhere(where interface{}, row []string, hMap map[string]int) (bool, error) {
if where == nil {
return true
return true, nil
}
ww, ok := where.(string)
wh, ok := where.(string)
if !ok {
return true
return true, nil
}
ev, err := types.GLang().NewEvaluable(ww)
ev, err := types.ExprLang.NewEvaluable(wh)
if err != nil {
panic(err)
return false, err
}
// prep payload
@@ -205,8 +206,8 @@ func checkWhere(where interface{}, row []string, hMap map[string]int) bool {
rr, err := ev.EvalBool(context.Background(), pr)
if err != nil {
panic(err)
return false, err
}
return rr
return rr, nil
}
pkg/ngImporter/types/eval.go → pkg/ngimporter/types/eval.go
View File
pkg/ngImporter/types/general.go → pkg/ngimporter/types/general.go
+4
View File
@@ -14,6 +14,10 @@ const (
EvalPrefix = "=EVL="
UserModHandle = "User"
MetaMapExt = ".map.json"
MetaJoinExt = ".join.json"
MetaValueExt = ".value.json"
)
var (
pkg/ngImporter/types/importNode.go → pkg/ngimporter/types/import_node.go
+4 -17
View File
@@ -141,7 +141,7 @@ func (n *ImportNode) Import(repoRecord repository.RecordRepository, users map[st
for _, p := range pps {
rtr = append(rtr, p)
// pass mapping object to the node's parend so it can migrate it's data
// pass mapping object to the node's parent so it can map handle dependency refs
p.addMap(fmt.Sprint(n.Module.ID), mapping)
p.LinkRemove(n)
}
@@ -236,7 +236,7 @@ func (n *ImportNode) removeIfPresent(rem *ImportNode, list []*ImportNode) []*Imp
return list
}
// traverses the graph and notifies us of any cycles
// SeekCycles finds cycles & calls the given function
func (n *ImportNode) SeekCycles(cycle func(n *ImportNode, to *ImportNode)) {
n.inPath = true
n.Visited = true
@@ -247,9 +247,6 @@ func (n *ImportNode) SeekCycles(cycle func(n *ImportNode, to *ImportNode)) {
}
for _, nn := range cc {
if n.Name == "client" {
}
if nn.inPath {
cycle(n, nn)
} else {
@@ -260,18 +257,6 @@ func (n *ImportNode) SeekCycles(cycle func(n *ImportNode, to *ImportNode)) {
n.inPath = false
}
func (n *ImportNode) DFS() {
n.inPath = true
for _, nn := range n.Children {
if !nn.inPath {
nn.DFS()
}
}
n.inPath = false
}
// clones the given node
func (n *ImportNode) clone() *ImportNode {
return &ImportNode{
@@ -288,6 +273,8 @@ func (n *ImportNode) clone() *ImportNode {
Namespace: n.Namespace,
Reader: n.Reader,
Header: n.Header,
FieldMap: n.FieldMap,
ValueMap: n.ValueMap,
}
}
pkg/ngImporter/types/ImportSource.go → pkg/ngimporter/types/import_source.go
View File