Add JSON(L) and CSV record exporting capabilities
API endpoints, CSV and JSON record encoders, dedicated svc/repo methods
This commit is contained in:
Denis Arh
@@ -0,0 +1,83 @@
|
||||
package encoder
|
||||
|
||||
type (
|
||||
multiple uint
|
||||
|
||||
field struct {
|
||||
name string
|
||||
encodeAllMulti bool
|
||||
}
|
||||
|
||||
FlatWriter interface {
|
||||
Write([]string) error
|
||||
Flush()
|
||||
}
|
||||
|
||||
StructuredEncoder interface {
|
||||
Encode(interface{}) error
|
||||
}
|
||||
|
||||
flatWriter struct {
|
||||
w FlatWriter
|
||||
ff []field
|
||||
}
|
||||
|
||||
structuredEncoder struct {
|
||||
w StructuredEncoder
|
||||
ff []field
|
||||
}
|
||||
)
|
||||
|
||||
func Field(name string) field {
|
||||
return field{name: name}
|
||||
}
|
||||
|
||||
func MakeFields(nn ...string) []field {
|
||||
ff := make([]field, len(nn))
|
||||
for i := range nn {
|
||||
ff[i] = field{name: nn[i]}
|
||||
}
|
||||
|
||||
return ff
|
||||
}
|
||||
|
||||
func MultiValueField(name string) field {
|
||||
return field{name: name, encodeAllMulti: true}
|
||||
}
|
||||
|
||||
func NewFlatWriter(w FlatWriter, header bool, ff ...field) *flatWriter {
|
||||
f := &flatWriter{
|
||||
w: w,
|
||||
ff: ff,
|
||||
}
|
||||
|
||||
if header {
|
||||
f.writeHeader()
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (enc flatWriter) Flush() {
|
||||
enc.w.Flush()
|
||||
}
|
||||
|
||||
func (enc flatWriter) writeHeader() {
|
||||
ss := make([]string, len(enc.ff))
|
||||
for i := range enc.ff {
|
||||
ss[i] = enc.ff[i].name
|
||||
}
|
||||
|
||||
_ = enc.w.Write(ss)
|
||||
}
|
||||
|
||||
func NewStructuredEncoder(w StructuredEncoder, ff ...field) *structuredEncoder {
|
||||
return &structuredEncoder{
|
||||
w: w,
|
||||
ff: ff,
|
||||
}
|
||||
}
|
||||
|
||||
func (enc structuredEncoder) Flush() {
|
||||
// noop
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package encoder
|
||||
@@ -0,0 +1,128 @@
|
||||
package encoder
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/compose/types"
|
||||
)
|
||||
|
||||
// Time formatter
|
||||
//
|
||||
// Takes ptr to time.Time so we can conver both cases (value + ptr)
|
||||
func fmtTime(tp *time.Time) string {
|
||||
if tp == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return tp.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func fmtUint64(u uint64) string {
|
||||
return strconv.FormatUint(u, 10)
|
||||
}
|
||||
|
||||
func (enc flatWriter) Record(r *types.Record) error {
|
||||
var out = make([]string, len(enc.ff))
|
||||
|
||||
for f, field := range enc.ff {
|
||||
switch field.name {
|
||||
case "recordID", "ID":
|
||||
out[f] = fmtUint64(r.ID)
|
||||
case "moduleID":
|
||||
out[f] = fmtUint64(r.ModuleID)
|
||||
case "namespaceID":
|
||||
out[f] = fmtUint64(r.NamespaceID)
|
||||
case "ownedBy":
|
||||
out[f] = fmtUint64(r.OwnedBy)
|
||||
case "createdBy":
|
||||
out[f] = fmtUint64(r.CreatedBy)
|
||||
case "createdAt":
|
||||
out[f] = fmtTime(&r.CreatedAt)
|
||||
case "updatedBy":
|
||||
out[f] = fmtUint64(r.UpdatedBy)
|
||||
case "updatedAt":
|
||||
out[f] = fmtTime(r.UpdatedAt)
|
||||
case "deletedBy":
|
||||
out[f] = fmtUint64(r.DeletedBy)
|
||||
case "deletedAt":
|
||||
out[f] = fmtTime(r.DeletedAt)
|
||||
default:
|
||||
vv := r.Values.FilterByName(field.name)
|
||||
// @todo support for field.encodeAllMulti
|
||||
if len(vv) > 0 {
|
||||
out[f] = vv[0].Value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
defer enc.w.Flush()
|
||||
|
||||
return enc.w.Write(out)
|
||||
}
|
||||
|
||||
func (enc structuredEncoder) Record(r *types.Record) error {
|
||||
var (
|
||||
// Exporter can choose fields so we need this buffer
|
||||
// to hold just what we need
|
||||
out = make(map[string]interface{})
|
||||
vv types.RecordValueSet
|
||||
c int
|
||||
)
|
||||
|
||||
for _, f := range enc.ff {
|
||||
switch f.name {
|
||||
case "recordID", "ID":
|
||||
out[f.name] = r.ID
|
||||
case "moduleID":
|
||||
out[f.name] = r.ModuleID
|
||||
case "namespaceID":
|
||||
out[f.name] = r.NamespaceID
|
||||
case "ownedBy":
|
||||
out[f.name] = r.OwnedBy
|
||||
case "createdBy":
|
||||
out[f.name] = r.CreatedBy
|
||||
case "createdAt":
|
||||
out[f.name] = fmtTime(&r.CreatedAt)
|
||||
case "updatedBy":
|
||||
out[f.name] = r.UpdatedBy
|
||||
case "updatedAt":
|
||||
if r.UpdatedAt == nil {
|
||||
out[f.name] = nil
|
||||
} else {
|
||||
out[f.name] = fmtTime(r.UpdatedAt)
|
||||
}
|
||||
|
||||
case "deletedBy":
|
||||
out[f.name] = r.DeletedBy
|
||||
case "deletedAt":
|
||||
if r.DeletedAt == nil {
|
||||
out[f.name] = nil
|
||||
} else {
|
||||
out[f.name] = fmtTime(r.DeletedAt)
|
||||
}
|
||||
|
||||
default:
|
||||
vv = r.Values.FilterByName(f.name)
|
||||
c = len(vv)
|
||||
|
||||
if c == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
if c == 1 {
|
||||
out[f.name] = vv[0].Value
|
||||
} else {
|
||||
multi := make([]string, c)
|
||||
|
||||
for n := range vv {
|
||||
multi[n] = vv[n].Value
|
||||
}
|
||||
|
||||
out[f.name] = multi
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return enc.w.Encode(out)
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package encoder
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cortezaproject/corteza-server/compose/types"
|
||||
"github.com/cortezaproject/corteza-server/internal/test"
|
||||
)
|
||||
|
||||
func Test_RecordEncoding(t *testing.T) {
|
||||
type args struct {
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
ff []field
|
||||
rr []*types.Record
|
||||
|
||||
flatResult string
|
||||
structResult string
|
||||
}{
|
||||
{
|
||||
name: "covering the basics",
|
||||
ff: MakeFields("recordID", "ownedBy", "createdAt", "deletedAt", "some-foo-field", "foo", "fff"),
|
||||
rr: []*types.Record{
|
||||
&types.Record{
|
||||
ID: 12345,
|
||||
OwnedBy: 12345,
|
||||
CreatedAt: time.Unix(1504976400, 0),
|
||||
},
|
||||
&types.Record{
|
||||
ID: 54321,
|
||||
OwnedBy: 12345,
|
||||
CreatedAt: time.Unix(12345, 0),
|
||||
Values: []*types.RecordValue{
|
||||
{
|
||||
Name: "foo",
|
||||
Value: "bar",
|
||||
},
|
||||
{
|
||||
Name: "fff",
|
||||
Value: "1",
|
||||
},
|
||||
{
|
||||
Name: "fff",
|
||||
Value: "2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
flatResult: `recordID,ownedBy,createdAt,deletedAt,some-foo-field,foo,fff` + "\n" +
|
||||
`12345,12345,2017-09-09T17:00:00Z,,,,` + "\n" +
|
||||
`54321,12345,1970-01-01T03:25:45Z,,,bar,1` + "\n",
|
||||
|
||||
structResult: `{"createdAt":"2017-09-09T17:00:00Z","deletedAt":null,"ownedBy":12345,"recordID":12345}` + "\n" +
|
||||
`{"createdAt":"1970-01-01T03:25:45Z","deletedAt":null,"fff":["1","2"],"foo":"bar","ownedBy":12345,"recordID":54321}` + "\n",
|
||||
},
|
||||
// TODO: Add test cases.
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name+" (csv)", func(t *testing.T) {
|
||||
buf := bytes.NewBuffer([]byte{})
|
||||
csvWriter := csv.NewWriter(buf)
|
||||
|
||||
fenc := NewFlatWriter(csvWriter, true, tt.ff...)
|
||||
for _, r := range tt.rr {
|
||||
if err := fenc.Record(r); err != nil {
|
||||
t.Errorf("unexpected error = %v,", err)
|
||||
}
|
||||
}
|
||||
|
||||
csvWriter.Flush()
|
||||
test.Assert(t,
|
||||
buf.String() == tt.flatResult,
|
||||
"Unexpected result: \n%s\n%s",
|
||||
buf.String(),
|
||||
tt.flatResult)
|
||||
})
|
||||
|
||||
t.Run(tt.name+" (json)", func(t *testing.T) {
|
||||
buf := bytes.NewBuffer([]byte{})
|
||||
jsonEnc := json.NewEncoder(buf)
|
||||
|
||||
senc := NewStructuredEncoder(jsonEnc, tt.ff...)
|
||||
for _, r := range tt.rr {
|
||||
if err := senc.Record(r); err != nil {
|
||||
t.Errorf("unexpected error = %v,", err)
|
||||
}
|
||||
}
|
||||
|
||||
test.Assert(t,
|
||||
buf.String() == tt.structResult,
|
||||
"Unexpected result: \n%s\n%s",
|
||||
buf.String(),
|
||||
tt.structResult)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user