Add record importing from json & csv

compose/decoder/decoder.go

@@ -0,0 +1,145 @@
+package decoder
+
+import (
+	"io"
+)
+
+type (
+	multiple uint
+
+	FlatReader interface {
+		Read() ([]string, error)
+	}
+
+	StructuredDecoder interface {
+		Decode(interface{}) error
+		More() bool
+	}
+
+	flatReader struct {
+		r      FlatReader
+		header []string
+		more   bool
+	}
+
+	structuredDecoder struct {
+		header []string
+		d      StructuredDecoder
+		buf    []map[string]interface{}
+	}
+
+	// callbacks
+	sdCallback func(map[string]interface{}) error
+	fdCallback func([]string) error
+)
+
+// flat reader
+func NewFlatReader(r FlatReader) *flatReader {
+	return &flatReader{
+		r:    r,
+		more: true,
+	}
+}
+
+func (dec *flatReader) get(fnc fdCallback) error {
+	v, err := dec.r.Read()
+	if err == io.EOF {
+		dec.more = false
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	return fnc(v)
+}
+
+func (dec *flatReader) walk(fnc fdCallback) error {
+	for dec.more {
+		if err := dec.get(fnc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (dec *flatReader) Header() []string {
+	if len(dec.header) > 0 {
+		return dec.header
+	}
+
+	dec.get(func(rtr []string) error {
+		dec.header = rtr
+		return nil
+	})
+
+	return dec.header
+}
+
+// structured decoder
+func NewStructuredDecoder(d StructuredDecoder) *structuredDecoder {
+	return &structuredDecoder{
+		d: d,
+	}
+}
+
+func (dec *structuredDecoder) get(fnc sdCallback) error {
+	if !dec.d.More() {
+		return nil
+	}
+
+	var tmp map[string]interface{}
+	err := dec.d.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	return fnc(tmp)
+}
+
+func (dec *structuredDecoder) exhaustBuffer(fnc sdCallback) error {
+	if dec.buf != nil {
+		for _, b := range dec.buf {
+			fnc(b)
+		}
+		dec.buf = nil
+	}
+	return nil
+}
+
+func (dec *structuredDecoder) walk(fnc sdCallback) error {
+	if err := dec.exhaustBuffer(fnc); err != nil {
+		return err
+	}
+
+	for dec.d.More() {
+		if err := dec.get(fnc); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (dec *structuredDecoder) Header() []string {
+	if len(dec.header) > 0 {
+		return dec.header
+	}
+
+	var tmp []string
+	dec.get(func(rtr map[string]interface{}) error {
+		// buffer first row or else it will be lost
+		dec.buf = append(dec.buf, rtr)
+
+		tmp = make([]string, len(rtr))
+		i := 0
+		for k := range rtr {
+			tmp[i] = k
+			i++
+		}
+
+		return nil
+	})
+
+	dec.header = tmp
+	return tmp
+}

compose/decoder/record.go

@@ -0,0 +1,131 @@
+package decoder
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/cortezaproject/corteza-server/compose/types"
+)
+
+type (
+	RecordCreator func(mod *types.Record) error
+)
+
+func fmtTime(tp string) (time.Time, error) {
+	return time.Parse(time.RFC3339, tp)
+}
+func fmtTimePtr(tp string) (*time.Time, error) {
+	t, err := fmtTime(tp)
+	if err != nil {
+		return nil, err
+	}
+	return &t, nil
+}
+
+func mapify(header []string, values []string) map[string]string {
+	rtr := make(map[string]string)
+	for i, v := range values {
+		rtr[header[i]] = v
+	}
+
+	return rtr
+}
+
+func setSystemField(r *types.Record, name, value string) (is bool, err error) {
+	switch name {
+	case "recordID", "ID":
+		r.ID, err = strconv.ParseUint(value, 10, 64)
+	case "moduleID":
+		r.ModuleID, err = strconv.ParseUint(value, 10, 64)
+	case "namespaceID":
+		r.NamespaceID, err = strconv.ParseUint(value, 10, 64)
+	case "ownedBy":
+		r.OwnedBy, err = strconv.ParseUint(value, 10, 64)
+	case "createdBy":
+		r.CreatedBy, err = strconv.ParseUint(value, 10, 64)
+	case "createdAt":
+		r.CreatedAt, err = fmtTime(value)
+	case "updatedBy":
+		r.UpdatedBy, err = strconv.ParseUint(value, 10, 64)
+	case "updatedAt":
+		r.UpdatedAt, err = fmtTimePtr(value)
+	case "deletedBy":
+		r.DeletedBy, err = strconv.ParseUint(value, 10, 64)
+	case "deletedAt":
+		r.DeletedAt, err = fmtTimePtr(value)
+	default:
+		return false, err
+	}
+	return true, err
+}
+
+func (dec flatReader) Records(fields map[string]string, Create RecordCreator) error {
+	header := dec.Header()
+
+	err := dec.walk(func(row []string) error {
+		mapped := mapify(header, row)
+		r := types.Record{}
+		rvs := types.RecordValueSet{}
+
+		i := 0
+		for imp, rec := range fields {
+			if rec == "" {
+				return errors.New("Can not import record: Record field not defined")
+			}
+
+			val := mapped[imp]
+			if system, err := setSystemField(&r, rec, val); err != nil {
+				return err
+			} else if !system {
+				rv := types.RecordValue{
+					Name:  rec,
+					Value: val,
+					Place: uint(i),
+				}
+				i++
+
+				rvs = append(rvs, &rv)
+			}
+		}
+
+		r.Values = rvs
+		return Create(&r)
+	})
+
+	return err
+}
+
+func (dec structuredDecoder) Records(fields map[string]string, Create RecordCreator) error {
+	err := dec.walk(func(entry map[string]interface{}) error {
+		r := types.Record{}
+		rvs := types.RecordValueSet{}
+
+		i := 0
+		for imp, rec := range fields {
+			if rec == "" {
+				return errors.New("Can not import record: Record field not defined")
+			}
+
+			val := fmt.Sprintf("%v", entry[imp])
+			if system, err := setSystemField(&r, rec, val); err != nil {
+				return err
+			} else if !system {
+				rv := types.RecordValue{
+					Name:  rec,
+					Value: val,
+					Place: uint(i),
+				}
+				i++
+
+				rvs = append(rvs, &rv)
+			}
+		}
+
+		r.Values = rvs
+		return Create(&r)
+	})
+
+	return err
+}