diff --git a/api/compose/spec.json b/api/compose/spec.json index 157ec81d4..0647e7564 100644 --- a/api/compose/spec.json +++ b/api/compose/spec.json @@ -683,16 +683,10 @@ "title": "Filtering condition" }, { - "name": "sort", - "type": "string", - "required": false, - "title": "Sort field (default id desc)" - }, - { - "name": "download", - "type": "bool", - "required": false, - "title": "Send headers to browser to trigger download/save-as" + "name": "fields", + "type": "[]string", + "required": true, + "title": "Fields to export" } ] } diff --git a/api/compose/spec/record.json b/api/compose/spec/record.json index 7f0d44cc3..f6aacf563 100644 --- a/api/compose/spec/record.json +++ b/api/compose/spec/record.json @@ -105,16 +105,10 @@ "type": "string" }, { - "name": "sort", - "required": false, - "title": "Sort field (default id desc)", - "type": "string" - }, - { - "name": "download", - "required": false, - "title": "Send headers to browser to trigger download/save-as", - "type": "bool" + "name": "fields", + "required": true, + "title": "Fields to export", + "type": "[]string" } ], "path": [ diff --git a/compose/encoder/encoder.go b/compose/encoder/encoder.go new file mode 100644 index 000000000..2d7a1206a --- /dev/null +++ b/compose/encoder/encoder.go @@ -0,0 +1,83 @@ +package encoder + +type ( + multiple uint + + field struct { + name string + encodeAllMulti bool + } + + FlatWriter interface { + Write([]string) error + Flush() + } + + StructuredEncoder interface { + Encode(interface{}) error + } + + flatWriter struct { + w FlatWriter + ff []field + } + + structuredEncoder struct { + w StructuredEncoder + ff []field + } +) + +func Field(name string) field { + return field{name: name} +} + +func MakeFields(nn ...string) []field { + ff := make([]field, len(nn)) + for i := range nn { + ff[i] = field{name: nn[i]} + } + + return ff +} + +func MultiValueField(name string) field { + return field{name: name, encodeAllMulti: true} +} + +func NewFlatWriter(w FlatWriter, header bool, ff ...field) *flatWriter { + f := &flatWriter{ + w: w, + ff: ff, + } + + if header { + f.writeHeader() + } + + return f +} + +func (enc flatWriter) Flush() { + enc.w.Flush() +} + +func (enc flatWriter) writeHeader() { + ss := make([]string, len(enc.ff)) + for i := range enc.ff { + ss[i] = enc.ff[i].name + } + + _ = enc.w.Write(ss) +} + +func NewStructuredEncoder(w StructuredEncoder, ff ...field) *structuredEncoder { + return &structuredEncoder{ + w: w, + ff: ff, + } +} + +func (enc structuredEncoder) Flush() { + // noop +} diff --git a/compose/encoder/encoder_xlsx.go b/compose/encoder/encoder_xlsx.go new file mode 100644 index 000000000..8a2c87f76 --- /dev/null +++ b/compose/encoder/encoder_xlsx.go @@ -0,0 +1 @@ +package encoder diff --git a/compose/encoder/record.go b/compose/encoder/record.go new file mode 100644 index 000000000..568bdb268 --- /dev/null +++ b/compose/encoder/record.go @@ -0,0 +1,128 @@ +package encoder + +import ( + "strconv" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" +) + +// Time formatter +// +// Takes ptr to time.Time so we can conver both cases (value + ptr) +func fmtTime(tp *time.Time) string { + if tp == nil { + return "" + } + + return tp.UTC().Format(time.RFC3339) +} + +func fmtUint64(u uint64) string { + return strconv.FormatUint(u, 10) +} + +func (enc flatWriter) Record(r *types.Record) error { + var out = make([]string, len(enc.ff)) + + for f, field := range enc.ff { + switch field.name { + case "recordID", "ID": + out[f] = fmtUint64(r.ID) + case "moduleID": + out[f] = fmtUint64(r.ModuleID) + case "namespaceID": + out[f] = fmtUint64(r.NamespaceID) + case "ownedBy": + out[f] = fmtUint64(r.OwnedBy) + case "createdBy": + out[f] = fmtUint64(r.CreatedBy) + case "createdAt": + out[f] = fmtTime(&r.CreatedAt) + case "updatedBy": + out[f] = fmtUint64(r.UpdatedBy) + case "updatedAt": + out[f] = fmtTime(r.UpdatedAt) + case "deletedBy": + out[f] = fmtUint64(r.DeletedBy) + case "deletedAt": + out[f] = fmtTime(r.DeletedAt) + default: + vv := r.Values.FilterByName(field.name) + // @todo support for field.encodeAllMulti + if len(vv) > 0 { + out[f] = vv[0].Value + } + } + } + + defer enc.w.Flush() + + return enc.w.Write(out) +} + +func (enc structuredEncoder) Record(r *types.Record) error { + var ( + // Exporter can choose fields so we need this buffer + // to hold just what we need + out = make(map[string]interface{}) + vv types.RecordValueSet + c int + ) + + for _, f := range enc.ff { + switch f.name { + case "recordID", "ID": + out[f.name] = r.ID + case "moduleID": + out[f.name] = r.ModuleID + case "namespaceID": + out[f.name] = r.NamespaceID + case "ownedBy": + out[f.name] = r.OwnedBy + case "createdBy": + out[f.name] = r.CreatedBy + case "createdAt": + out[f.name] = fmtTime(&r.CreatedAt) + case "updatedBy": + out[f.name] = r.UpdatedBy + case "updatedAt": + if r.UpdatedAt == nil { + out[f.name] = nil + } else { + out[f.name] = fmtTime(r.UpdatedAt) + } + + case "deletedBy": + out[f.name] = r.DeletedBy + case "deletedAt": + if r.DeletedAt == nil { + out[f.name] = nil + } else { + out[f.name] = fmtTime(r.DeletedAt) + } + + default: + vv = r.Values.FilterByName(f.name) + c = len(vv) + + if c == 0 { + break + } + + if c == 1 { + out[f.name] = vv[0].Value + } else { + multi := make([]string, c) + + for n := range vv { + multi[n] = vv[n].Value + } + + out[f.name] = multi + } + } + } + + return enc.w.Encode(out) +} diff --git a/compose/encoder/record_test.go b/compose/encoder/record_test.go new file mode 100644 index 000000000..7ac1d4507 --- /dev/null +++ b/compose/encoder/record_test.go @@ -0,0 +1,103 @@ +package encoder + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "testing" + "time" + + "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/internal/test" +) + +func Test_RecordEncoding(t *testing.T) { + type args struct { + } + tests := []struct { + name string + ff []field + rr []*types.Record + + flatResult string + structResult string + }{ + { + name: "covering the basics", + ff: MakeFields("recordID", "ownedBy", "createdAt", "deletedAt", "some-foo-field", "foo", "fff"), + rr: []*types.Record{ + &types.Record{ + ID: 12345, + OwnedBy: 12345, + CreatedAt: time.Unix(1504976400, 0), + }, + &types.Record{ + ID: 54321, + OwnedBy: 12345, + CreatedAt: time.Unix(12345, 0), + Values: []*types.RecordValue{ + { + Name: "foo", + Value: "bar", + }, + { + Name: "fff", + Value: "1", + }, + { + Name: "fff", + Value: "2", + }, + }, + }, + }, + + flatResult: `recordID,ownedBy,createdAt,deletedAt,some-foo-field,foo,fff` + "\n" + + `12345,12345,2017-09-09T17:00:00Z,,,,` + "\n" + + `54321,12345,1970-01-01T03:25:45Z,,,bar,1` + "\n", + + structResult: `{"createdAt":"2017-09-09T17:00:00Z","deletedAt":null,"ownedBy":12345,"recordID":12345}` + "\n" + + `{"createdAt":"1970-01-01T03:25:45Z","deletedAt":null,"fff":["1","2"],"foo":"bar","ownedBy":12345,"recordID":54321}` + "\n", + }, + // TODO: Add test cases. + } + + for _, tt := range tests { + t.Run(tt.name+" (csv)", func(t *testing.T) { + buf := bytes.NewBuffer([]byte{}) + csvWriter := csv.NewWriter(buf) + + fenc := NewFlatWriter(csvWriter, true, tt.ff...) + for _, r := range tt.rr { + if err := fenc.Record(r); err != nil { + t.Errorf("unexpected error = %v,", err) + } + } + + csvWriter.Flush() + test.Assert(t, + buf.String() == tt.flatResult, + "Unexpected result: \n%s\n%s", + buf.String(), + tt.flatResult) + }) + + t.Run(tt.name+" (json)", func(t *testing.T) { + buf := bytes.NewBuffer([]byte{}) + jsonEnc := json.NewEncoder(buf) + + senc := NewStructuredEncoder(jsonEnc, tt.ff...) + for _, r := range tt.rr { + if err := senc.Record(r); err != nil { + t.Errorf("unexpected error = %v,", err) + } + } + + test.Assert(t, + buf.String() == tt.structResult, + "Unexpected result: \n%s\n%s", + buf.String(), + tt.structResult) + }) + } +} diff --git a/compose/rest/record.go b/compose/rest/record.go index 7c9e4a988..78200ef1a 100644 --- a/compose/rest/record.go +++ b/compose/rest/record.go @@ -2,17 +2,21 @@ package rest import ( "context" + "encoding/csv" "encoding/json" "fmt" "net/http" + "strings" "github.com/titpetric/factory/resputil" "github.com/pkg/errors" + "github.com/cortezaproject/corteza-server/compose/encoder" "github.com/cortezaproject/corteza-server/compose/internal/service" "github.com/cortezaproject/corteza-server/compose/rest/request" "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/rh" ) var _ = errors.Wrap @@ -71,8 +75,8 @@ func (ctrl *Record) List(ctx context.Context, r *request.RecordList) (interface{ ModuleID: r.ModuleID, Filter: r.Filter, Sort: r.Sort, - PerPage: r.PerPage, - Page: r.Page, + + PageFilter: rh.Paging(r.Page, r.PerPage), }) return ctrl.makeFilterPayload(ctx, m, rr, filter, err) @@ -158,41 +162,64 @@ func (ctrl *Record) Upload(ctx context.Context, r *request.RecordUpload) (interf } func (ctrl *Record) Export(ctx context.Context, r *request.RecordExport) (interface{}, error) { - var ( - m *types.Module - err error + type ( + // ad-hoc interface for our encoder + Encoder interface { + service.Encoder + Flush() + } ) - if m, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil { + var ( + err error + + // Record encoder + recordEncoder Encoder + + filename = fmt.Sprintf("; filename=%s.%s", r.Filename, r.Ext) + + f = types.RecordFilter{ + NamespaceID: r.NamespaceID, + ModuleID: r.ModuleID, + Filter: r.Filter, + } + + contentType string + ) + + // Access control. + if _, err = ctrl.module.With(ctx).FindByID(r.NamespaceID, r.ModuleID); err != nil { return nil, err } - _ = m - - // will probably have to rewrite exporting into something more optimal: - // maybe pass encoding function/callback directly - rr, _, err := ctrl.record.With(ctx).Find(types.RecordFilter{ - NamespaceID: r.NamespaceID, - ModuleID: r.ModuleID, - Filter: r.Filter, - Sort: r.Sort, - }) + if len(r.Fields) == 1 { + r.Fields = strings.Split(r.Fields[0], ",") + } return func(w http.ResponseWriter, req *http.Request) { - var ( - enc = json.NewEncoder(w) - filename = fmt.Sprintf("; filename=%s.%s", r.Filename, r.Ext) - ) + ff := encoder.MakeFields(r.Fields...) - if r.Download { - w.Header().Add("Content-Disposition", "attachment"+filename) - } else { - w.Header().Add("Content-Disposition", "inline"+filename) + switch strings.ToLower(r.Ext) { + case "json", "jsonl", "ldjson", "ndjson": + contentType = "application/jsonl" + recordEncoder = encoder.NewStructuredEncoder(json.NewEncoder(w), ff...) + + case "csv": + encoder = encoder2.NewFlatWriter(csv.NewWriter(w), true, encoder2.MakeFields(r.Fields...)...) + default: + http.Error(w, "unsupported format ("+r.Ext+")", http.StatusBadRequest) + return } - _ = rr.Walk(func(record *types.Record) error { - return enc.Encode(record) - }) + w.Header().Add("Content-Type", contentType) + w.Header().Add("Content-Disposition", "attachment"+filename) + + if err = ctrl.record.With(ctx).Export(f, recordEncoder); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + recordEncoder.Flush() }, nil } diff --git a/compose/rest/request/record.go b/compose/rest/request/record.go index c29dc2f58..8cc5b9d77 100644 --- a/compose/rest/request/record.go +++ b/compose/rest/request/record.go @@ -178,8 +178,7 @@ var _ RequestFiller = NewRecordList() // Record export request parameters type RecordExport struct { Filter string - Sort string - Download bool + Fields []string Filename string Ext string NamespaceID uint64 `json:",string"` @@ -194,8 +193,7 @@ func (r RecordExport) Auditable() map[string]interface{} { var out = map[string]interface{}{} out["filter"] = r.Filter - out["sort"] = r.Sort - out["download"] = r.Download + out["fields"] = r.Fields out["filename"] = r.Filename out["ext"] = r.Ext out["namespaceID"] = r.NamespaceID @@ -234,12 +232,13 @@ func (r *RecordExport) Fill(req *http.Request) (err error) { if val, ok := get["filter"]; ok { r.Filter = val } - if val, ok := get["sort"]; ok { - r.Sort = val - } - if val, ok := get["download"]; ok { - r.Download = parseBool(val) + + if val, ok := urlQuery["fields[]"]; ok { + r.Fields = parseStrings(val) + } else if val, ok = urlQuery["fields"]; ok { + r.Fields = parseStrings(val) } + r.Filename = chi.URLParam(req, "filename") r.Ext = chi.URLParam(req, "ext") r.NamespaceID = parseUInt64(chi.URLParam(req, "namespaceID")) diff --git a/docs/compose/README.md b/docs/compose/README.md index 5275b6980..7fdcbe9a5 100644 --- a/docs/compose/README.md +++ b/docs/compose/README.md @@ -738,8 +738,7 @@ Compose records | Parameter | Type | Method | Description | Default | Required? | | --------- | ---- | ------ | ----------- | ------- | --------- | | filter | string | GET | Filtering condition | N/A | NO | -| sort | string | GET | Sort field (default id desc) | N/A | NO | -| download | bool | GET | Send headers to browser to trigger download/save-as | N/A | NO | +| fields | []string | GET | Fields to export | N/A | YES | | filename | string | PATH | Filename to use | N/A | NO | | ext | string | PATH | Export format | N/A | YES | | namespaceID | uint64 | PATH | Namespace ID | N/A | YES |