DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Add JSON(L) and CSV record exporting capabilities

Browse Source 
API endpoints, CSV and JSON record encoders, dedicated svc/repo methods
This commit is contained in:
Denis Arh
2019-08-15 12:39:31 +02:00
parent 5ee1a90655
commit 5423cf43c8
9 changed files with 386 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/compose/spec.json
+4 -10
View File
@@ -683,16 +683,10 @@
"title": "Filtering condition"
},
{
"name": "sort",
"type": "string",
"required": false,
"title": "Sort field (default id desc)"
},
{
"name": "download",
"type": "bool",
"required": false,
"title": "Send headers to browser to trigger download/save-as"
"name": "fields",
"type": "[]string",
"required": true,
"title": "Fields to export"
}
]
}
api/compose/spec/record.json
+4 -10
View File
@@ -105,16 +105,10 @@
"type": "string"
},
{
"name": "sort",
"required": false,
"title": "Sort field (default id desc)",
"type": "string"
},
{
"name": "download",
"required": false,
"title": "Send headers to browser to trigger download/save-as",
"type": "bool"
"name": "fields",
"required": true,
"title": "Fields to export",
"type": "[]string"
}
],
"path": [
compose/encoder/encoder.go
+83
View File
@@ -0,0 +1,83 @@
package encoder
type (
multiple uint
field struct {
name string
encodeAllMulti bool
}
FlatWriter interface {
Write([]string) error
Flush()
}
StructuredEncoder interface {
Encode(interface{}) error
}
flatWriter struct {
w FlatWriter
ff []field
}
structuredEncoder struct {
w StructuredEncoder
ff []field
}
)
func Field(name string) field {
return field{name: name}
}
func MakeFields(nn ...string) []field {
ff := make([]field, len(nn))
for i := range nn {
ff[i] = field{name: nn[i]}
}
return ff
}
func MultiValueField(name string) field {
return field{name: name, encodeAllMulti: true}
}
func NewFlatWriter(w FlatWriter, header bool, ff ...field) *flatWriter {
f := &flatWriter{
w: w,
ff: ff,
}
if header {
f.writeHeader()
}
return f
}
func (enc flatWriter) Flush() {
enc.w.Flush()
}
func (enc flatWriter) writeHeader() {
ss := make([]string, len(enc.ff))
for i := range enc.ff {
ss[i] = enc.ff[i].name
}
_ = enc.w.Write(ss)
}
func NewStructuredEncoder(w StructuredEncoder, ff ...field) *structuredEncoder {
return &structuredEncoder{
w: w,
ff: ff,
}
}
func (enc structuredEncoder) Flush() {
// noop
}
compose/encoder/encoder_xlsx.go
+1
View File
@@ -0,0 +1 @@
package encoder
compose/encoder/record.go
+128
View File
@@ -0,0 +1,128 @@
package encoder
import (
"strconv"
"time"
"github.com/cortezaproject/corteza-server/compose/types"
)
// Time formatter
//
// Takes ptr to time.Time so we can conver both cases (value + ptr)
func fmtTime(tp *time.Time) string {
if tp == nil {
return ""
}
return tp.UTC().Format(time.RFC3339)
}
func fmtUint64(u uint64) string {
return strconv.FormatUint(u, 10)
}
func (enc flatWriter) Record(r *types.Record) error {
var out = make([]string, len(enc.ff))
for f, field := range enc.ff {
switch field.name {
case "recordID", "ID":
out[f] = fmtUint64(r.ID)
case "moduleID":
out[f] = fmtUint64(r.ModuleID)
case "namespaceID":
out[f] = fmtUint64(r.NamespaceID)
case "ownedBy":
out[f] = fmtUint64(r.OwnedBy)
case "createdBy":
out[f] = fmtUint64(r.CreatedBy)
case "createdAt":
out[f] = fmtTime(&r.CreatedAt)
case "updatedBy":
out[f] = fmtUint64(r.UpdatedBy)
case "updatedAt":
out[f] = fmtTime(r.UpdatedAt)
case "deletedBy":
out[f] = fmtUint64(r.DeletedBy)
case "deletedAt":
out[f] = fmtTime(r.DeletedAt)
default:
vv := r.Values.FilterByName(field.name)
// @todo support for field.encodeAllMulti
if len(vv) > 0 {
out[f] = vv[0].Value
}
}
}
defer enc.w.Flush()
return enc.w.Write(out)
}
func (enc structuredEncoder) Record(r *types.Record) error {
var (
// Exporter can choose fields so we need this buffer
// to hold just what we need
out = make(map[string]interface{})
vv types.RecordValueSet
c int
)
for _, f := range enc.ff {
switch f.name {
case "recordID", "ID":
out[f.name] = r.ID
case "moduleID":
out[f.name] = r.ModuleID
case "namespaceID":
out[f.name] = r.NamespaceID
case "ownedBy":
out[f.name] = r.OwnedBy
case "createdBy":
out[f.name] = r.CreatedBy
case "createdAt":
out[f.name] = fmtTime(&r.CreatedAt)
case "updatedBy":
out[f.name] = r.UpdatedBy
case "updatedAt":
if r.UpdatedAt == nil {
out[f.name] = nil
} else {
out[f.name] = fmtTime(r.UpdatedAt)
}
case "deletedBy":
out[f.name] = r.DeletedBy
case "deletedAt":
if r.DeletedAt == nil {
out[f.name] = nil
} else {
out[f.name] = fmtTime(r.DeletedAt)
}
default:
vv = r.Values.FilterByName(f.name)
c = len(vv)
if c == 0 {
break
}
if c == 1 {
out[f.name] = vv[0].Value
} else {
multi := make([]string, c)
for n := range vv {
multi[n] = vv[n].Value
}
out[f.name] = multi
}
}
}
return enc.w.Encode(out)
}
compose/encoder/record_test.go
+103
View File
@@ -0,0 +1,103 @@
package encoder
import (
"bytes"
"encoding/csv"
"encoding/json"
"testing"
"time"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/internal/test"
)
func Test_RecordEncoding(t *testing.T) {
type args struct {
}
tests := []struct {
name string
ff []field
rr []*types.Record
flatResult string
structResult string
}{
{
name: "covering the basics",
ff: MakeFields("recordID", "ownedBy", "createdAt", "deletedAt", "some-foo-field", "foo", "fff"),
rr: []*types.Record{
&types.Record{
ID: 12345,
OwnedBy: 12345,
CreatedAt: time.Unix(1504976400, 0),
},
&types.Record{
ID: 54321,
OwnedBy: 12345,
CreatedAt: time.Unix(12345, 0),
Values: []*types.RecordValue{
{
Name: "foo",
Value: "bar",
},
{
Name: "fff",
Value: "1",
},
{
Name: "fff",
Value: "2",
},
},
},
},
flatResult: `recordID,ownedBy,createdAt,deletedAt,some-foo-field,foo,fff` + "\n" +
`12345,12345,2017-09-09T17:00:00Z,,,,` + "\n" +
`54321,12345,1970-01-01T03:25:45Z,,,bar,1` + "\n",
structResult: `{"createdAt":"2017-09-09T17:00:00Z","deletedAt":null,"ownedBy":12345,"recordID":12345}` + "\n" +
`{"createdAt":"1970-01-01T03:25:45Z","deletedAt":null,"fff":["1","2"],"foo":"bar","ownedBy":12345,"recordID":54321}` + "\n",
},
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name+" (csv)", func(t *testing.T) {
buf := bytes.NewBuffer([]byte{})
csvWriter := csv.NewWriter(buf)
fenc := NewFlatWriter(csvWriter, true, tt.ff...)
for _, r := range tt.rr {
if err := fenc.Record(r); err != nil {
t.Errorf("unexpected error = %v,", err)
}
}
csvWriter.Flush()
test.Assert(t,
buf.String() == tt.flatResult,
"Unexpected result: \n%s\n%s",
buf.String(),
tt.flatResult)
})
t.Run(tt.name+" (json)", func(t *testing.T) {
buf := bytes.NewBuffer([]byte{})
jsonEnc := json.NewEncoder(buf)
senc := NewStructuredEncoder(jsonEnc, tt.ff...)
for _, r := range tt.rr {
if err := senc.Record(r); err != nil {
t.Errorf("unexpected error = %v,", err)
}
}
test.Assert(t,
buf.String() == tt.structResult,
"Unexpected result: \n%s\n%s",
buf.String(),
tt.structResult)
})
}
}
compose/rest/record.go
+54 -27
View File
@@ -2,17 +2,21 @@ package rest
import (
"context"
"encoding/csv"
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/titpetric/factory/resputil"
"github.com/pkg/errors"
"github.com/cortezaproject/corteza-server/compose/encoder"
"github.com/cortezaproject/corteza-server/compose/internal/service"
"github.com/cortezaproject/corteza-server/compose/rest/request"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/rh"
)
var _ = errors.Wrap
@@ -71,8 +75,8 @@ func (ctrl *Record) List(ctx context.Context, r *request.RecordList) (interface{
ModuleID: r.ModuleID,
Filter: r.Filter,
Sort: r.Sort,
PerPage: r.PerPage,
Page: r.Page,
PageFilter: rh.Paging(r.Page, r.PerPage),
})
return ctrl.makeFilterPayload(ctx, m, rr, filter, err)
@@ -158,41 +162,64 @@ func (ctrl *Record) Upload(ctx context.Context, r *request.RecordUpload) (interf
}
func (ctrl *Record) Export(ctx context.Context, r *request.RecordExport) (interface{}, error) {
var (
m *types.Module
err error
type (
// ad-hoc interface for our encoder
Encoder interface {
service.Encoder
Flush()
}
)
if m, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil {
var (
err error
// Record encoder
recordEncoder Encoder
filename = fmt.Sprintf("; filename=%s.%s", r.Filename, r.Ext)
f = types.RecordFilter{
NamespaceID: r.NamespaceID,
ModuleID: r.ModuleID,
Filter: r.Filter,
}
contentType string
)
// Access control.
if _, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil {
return nil, err
}
_ = m
// will probably have to rewrite exporting into something more optimal:
// maybe pass encoding function/callback directly
rr, _, err := ctrl.record.With(ctx).Find(types.RecordFilter{
NamespaceID: r.NamespaceID,
ModuleID: r.ModuleID,
Filter: r.Filter,
Sort: r.Sort,
})
if len(r.Fields) == 1 {
r.Fields = strings.Split(r.Fields[0], ",")
}
return func(w http.ResponseWriter, req *http.Request) {
var (
enc = json.NewEncoder(w)
filename = fmt.Sprintf("; filename=%s.%s", r.Filename, r.Ext)
)
ff := encoder.MakeFields(r.Fields...)
if r.Download {
w.Header().Add("Content-Disposition", "attachment"+filename)
} else {
w.Header().Add("Content-Disposition", "inline"+filename)
switch strings.ToLower(r.Ext) {
case "json", "jsonl", "ldjson", "ndjson":
contentType = "application/jsonl"
recordEncoder = encoder.NewStructuredEncoder(json.NewEncoder(w), ff...)
case "csv":
encoder = encoder2.NewFlatWriter(csv.NewWriter(w), true, encoder2.MakeFields(r.Fields...)...)
default:
http.Error(w, "unsupported format ("+r.Ext+")", http.StatusBadRequest)
return
}
_ = rr.Walk(func(record *types.Record) error {
return enc.Encode(record)
})
w.Header().Add("Content-Type", contentType)
w.Header().Add("Content-Disposition", "attachment"+filename)
if err = ctrl.record.With(ctx).Export(f, recordEncoder); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
recordEncoder.Flush()
}, nil
}
compose/rest/request/record.go
+8 -9
View File
@@ -178,8 +178,7 @@ var _ RequestFiller = NewRecordList()
// Record export request parameters
type RecordExport struct {
Filter string
Sort string
Download bool
Fields []string
Filename string
Ext string
NamespaceID uint64 `json:",string"`
@@ -194,8 +193,7 @@ func (r RecordExport) Auditable() map[string]interface{} {
var out = map[string]interface{}{}
out["filter"] = r.Filter
out["sort"] = r.Sort
out["download"] = r.Download
out["fields"] = r.Fields
out["filename"] = r.Filename
out["ext"] = r.Ext
out["namespaceID"] = r.NamespaceID
@@ -234,12 +232,13 @@ func (r *RecordExport) Fill(req *http.Request) (err error) {
if val, ok := get["filter"]; ok {
r.Filter = val
}
if val, ok := get["sort"]; ok {
r.Sort = val
}
if val, ok := get["download"]; ok {
r.Download = parseBool(val)
if val, ok := urlQuery["fields[]"]; ok {
r.Fields = parseStrings(val)
} else if val, ok = urlQuery["fields"]; ok {
r.Fields = parseStrings(val)
}
r.Filename = chi.URLParam(req, "filename")
r.Ext = chi.URLParam(req, "ext")
r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID"))
docs/compose/README.md
+1 -2
View File
@@ -738,8 +738,7 @@ Compose records
| Parameter | Type | Method | Description | Default | Required? |
| --------- | ---- | ------ | ----------- | ------- | --------- |
| filter | string | GET | Filtering condition | N/A | NO |
| sort | string | GET | Sort field (default id desc) | N/A | NO |
| download | bool | GET | Send headers to browser to trigger download/save-as | N/A | NO |
| fields | []string | GET | Fields to export | N/A | YES |
| filename | string | PATH | Filename to use | N/A | NO |
| ext | string | PATH | Export format | N/A | YES |
| namespaceID | uint64 | PATH | Namespace ID | N/A | YES |