DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Fix ordering records by values, record organizer

Browse Source
This commit is contained in:
Denis Arh
2020-09-09 13:10:40 +02:00
parent 15af5de7c4
commit edbeb57f48
17 changed files with 168 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/service/record.go
+18 -15
View File
@@ -979,17 +979,16 @@ func (svc record) Organize(namespaceID, moduleID, recordID uint64, posField, pos
return store.Tx(svc.ctx, svc.store, func(ctx context.Context, s store.Storer) error {
if len(recordValues) > 0 {
svc.recordInfoUpdate(r)
if err = store.UpdateComposeRecord(ctx, s, m, r); err != nil {
return err
}
panic("refactor")
//if err = svc.recordRepo.PartialUpdateValues(recordValues...); err != nil {
//svc.recordInfoUpdate(r)
//if err = store.UpdateComposeRecord(ctx, s, m, r); err != nil {
// return err
//}
}
if err = store.PartialComposeRecordValueUpdate(ctx, s, m, recordValues...); err != nil {
return err
}
if reorderingRecords {
var (
set types.RecordSet
@@ -1021,17 +1020,21 @@ func (svc record) Organize(namespaceID, moduleID, recordID uint64, posField, pos
}
// Update value on each record
return set.Walk(func(r *types.Record) error {
var vv = make([]*types.RecordValue, 0, len(set))
_ = set.Walk(func(r *types.Record) error {
recordOrderPlace++
vv = append(vv, &types.RecordValue{
RecordID: r.ID,
Name: posField,
Value: strconv.FormatUint(recordOrderPlace, 10),
})
// Update each and every set
panic("refactor")
//return svc.recordRepo.PartialUpdateValues(&types.RecordValue{
// RecordID: r.ID,
// Name: posField,
// Value: strconv.FormatUint(recordOrderPlace, 10),
//})
return nil
})
if err = store.PartialComposeRecordValueUpdate(ctx, s, m, vv...); err != nil {
return err
}
}
return nil
compose/service/service.go
+11 -1
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"errors"
"fmt"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/actionlog"
"github.com/cortezaproject/corteza-server/pkg/corredor"
@@ -16,6 +17,7 @@ import (
ngStore "github.com/cortezaproject/corteza-server/store"
systemService "github.com/cortezaproject/corteza-server/system/service"
"go.uber.org/zap"
"strconv"
"time"
)
@@ -176,7 +178,15 @@ func RegisterIteratorProviders() {
rf := types.RecordFilter{Query: f["query"]}
rf.Sort.Set(f["sort"])
panic("refactor")
if lString, has := f["limit"]; has {
if limit, err := strconv.ParseUint(lString, 10, 32); err != nil {
return fmt.Errorf("can not parse iterator limit param: %w", err)
} else {
rf.Limit = uint(limit)
}
}
//panic("refactor")
//rf.Paging.Limit =
//rf.ParsePagination(f)
store/compose_records.gen.go
+7
View File
@@ -34,6 +34,9 @@ type (
// ComposeRecordReport (custom function)
ComposeRecordReport(ctx context.Context, _mod *types.Module, _metrics string, _dimensions string, _filters string) ([]map[string]interface{}, error)
// PartialComposeRecordValueUpdate (custom function)
PartialComposeRecordValueUpdate(ctx context.Context, _mod *types.Module, _values ...*types.RecordValue) error
}
)
@@ -89,3 +92,7 @@ func TruncateComposeRecords(ctx context.Context, s ComposeRecords, _mod *types.M
func ComposeRecordReport(ctx context.Context, s ComposeRecords, _mod *types.Module, _metrics string, _dimensions string, _filters string) ([]map[string]interface{}, error) {
return s.ComposeRecordReport(ctx, _mod, _metrics, _dimensions, _filters)
}
func PartialComposeRecordValueUpdate(ctx context.Context, s ComposeRecords, _mod *types.Module, _values ...*types.RecordValue) error {
return s.PartialComposeRecordValueUpdate(ctx, _mod, _values...)
}
store/compose_records.yaml
+6
View File
@@ -25,6 +25,12 @@ functions:
- { name: filters, type: string }
return: [ "[]map[string]interface{}", error ]
- name: PartialComposeRecordValueUpdate
arguments:
- { name: mod, type: "*types.Module" }
- { name: values, type: ...*types.RecordValue }
return: [ error ]
lookups:
- fields: [ ID ]
export: false
store/mysql/sql_typecasters.go
+7 -8
View File
@@ -2,23 +2,22 @@ package mysql
import (
"fmt"
"github.com/cortezaproject/corteza-server/pkg/ql"
"github.com/cortezaproject/corteza-server/store/rdbms"
)
func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, i ql.Ident) (ql.Ident, error) {
func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, ident string) (string, error) {
switch true {
case field.IsBoolean():
i.Value = fmt.Sprintf("(rv_%s.value NOT IN ('', '0', 'false', 'f', 'FALSE', 'F', false))", i.Value)
return fmt.Sprintf("(rv_%s.value NOT IN ('', '0', 'false', 'f', 'FALSE', 'F', false))", ident), nil
case field.IsNumeric():
i.Value = fmt.Sprintf("CAST(rv_%s.value AS SIGNED)", i.Value)
return fmt.Sprintf("CAST(rv_%s.value AS SIGNED)", ident), nil
case field.IsDateTime():
i.Value = fmt.Sprintf("CAST(rv_%s.value AS DATETIME)", i.Value)
return fmt.Sprintf("CAST(rv_%s.value AS DATETIME)", ident), nil
case field.IsRef():
i.Value = fmt.Sprintf("rv_%s.ref ", i.Value)
return fmt.Sprintf("rv_%s.ref ", ident), nil
default:
i.Value = fmt.Sprintf("rv_%s.value ", i.Value)
return fmt.Sprintf("rv_%s.value ", ident), nil
}
return i, nil
return ident, nil
}
store/pgsql/sql_typecasters.go
+7 -8
View File
@@ -2,23 +2,22 @@ package pgsql
import (
"fmt"
"github.com/cortezaproject/corteza-server/pkg/ql"
"github.com/cortezaproject/corteza-server/store/rdbms"
)
func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, i ql.Ident) (ql.Ident, error) {
func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, ident string) (string, error) {
switch true {
case field.IsBoolean():
i.Value = fmt.Sprintf("rv_%s.value NOT IN ('', '0', 'false', 'f', 'FALSE', 'F', false)", i.Value)
return fmt.Sprintf("rv_%s.value NOT IN ('', '0', 'false', 'f', 'FALSE', 'F', false)", ident), nil
case field.IsNumeric():
i.Value = fmt.Sprintf("rv_%s.value::NUMERIC", i.Value)
return fmt.Sprintf("rv_%s.value::NUMERIC", ident), nil
case field.IsDateTime():
i.Value = fmt.Sprintf("rv_%s.value::TIMESTAMP", i.Value)
return fmt.Sprintf("rv_%s.value::TIMESTAMP", ident), nil
case field.IsRef():
i.Value = fmt.Sprintf("rv_%s.ref ", i.Value)
return fmt.Sprintf("rv_%s.ref ", ident), nil
default:
i.Value = fmt.Sprintf("rv_%s.value ", i.Value)
return fmt.Sprintf("rv_%s.value ", ident), nil
}
return i, nil
return ident, nil
}
store/rdbms/builder.go
+3
View File
@@ -16,6 +16,7 @@ import (
// https://sqlite.org/lang_UPSERT.html
func UpsertBuilder(cfg *Config, table string, payload store.Payload, columns ...string) (squirrel.InsertBuilder, error) {
var (
updateCond = squirrel.Eq{}
updatePayload = store.Payload{}
// where to cutoff the update query
@@ -29,12 +30,14 @@ func UpsertBuilder(cfg *Config, table string, payload store.Payload, columns ...
for _, c := range columns {
if _, has := updatePayload[c]; has {
delete(updatePayload, c)
updateCond[c] = payload[c]
}
}
sql, args, err := squirrel.
Update(table).
SetMap(updatePayload).
//Where(updateCond).
ToSql()
if err != nil {
store/rdbms/compose_record_value.go
+14
View File
@@ -44,3 +44,17 @@ func (s Store) ComposeRecordValueRefLookup(ctx context.Context, m *types.Module,
return recordID, nil
}
// PartialComposeRecordValueUpdate2 updates specific record values across multiple records
func (s Store) PartialComposeRecordValueUpdate(ctx context.Context, m *types.Module, vv ...*types.RecordValue) (err error) {
{
// handle standard record-value storage
for _, v := range vv {
if err = s.execUpsertComposeRecordValues(ctx, s.internalComposeRecordValueEncoder(v)); err != nil {
return
}
}
}
return nil
}
store/rdbms/compose_records.go
+10 -6
View File
@@ -317,13 +317,19 @@ func (s Store) composeRecordsSorter(m *types.Module, q squirrel.SelectBuilder, s
)
for i, c := range sort {
var err error
if sortable[c.Column] {
sqlSort[i] = sort[i].Column
} else if m.Fields.HasName(c.Column) {
sqlSort[i] = c.Column
} else if f := m.Fields.FindByName(c.Column); f != nil {
//sqlSort[i] = fmt.Sprintf("%s%s.value", composeRecordValueAliasPfx, sort[i].Column)
sqlSort[i] = fmt.Sprintf("COALESCE(%s%s.value, '')", composeRecordValueAliasPfx, sort[i].Column)
// sqlSort[i] = fmt.Sprintf("COALESCE(%s%s.value, '')", composeRecordValueAliasPfx + sort[i].Column)
sqlSort[i], err = s.config.CastModuleFieldToColumnType(f, c.Column)
} else {
return q, fmt.Errorf("could not sort by unknown column: %s", c.Column)
err = fmt.Errorf("could not sort by unknown column: %s", c.Column)
}
if err != nil {
return q, err
}
if sort[i].Descending {
@@ -360,8 +366,6 @@ func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string)
} else {
cursor.Set(fmt.Sprintf("%s%s.value", composeRecordValueAliasPfx, c), nil, false)
}
//cursor.Set(fmt.Sprintf("COALESCE(%s%s.value, '')", composeRecordValueAliasPfx, c), value, false)
}
}
}
store/rdbms/rdbms.go
+3 -1
View File
@@ -364,7 +364,9 @@ func (s Store) SqlFunctionHandler(f ql.Function) (ql.ASTNode, error) {
// FieldToColumnTypeCaster calls configured field type caster if set
// otherwise returns passed arguments directly
func (s Store) FieldToColumnTypeCaster(f ModuleFieldTypeDetector, i ql.Ident) (ql.Ident, error) {
return s.config.CastModuleFieldToColumnType(f, i)
var err error
i.Value, err = s.config.CastModuleFieldToColumnType(f, i.Value)
return i, err
}
// tx begins a new db transaction and handles it's retries when possible
store/rdbms/rdbms_config.go
+1 -1
View File
@@ -107,7 +107,7 @@ type (
// Functions are used in filters and aggregations
SqlFunctionHandler func(f ql.Function) (ql.ASTNode, error)
CastModuleFieldToColumnType func(field ModuleFieldTypeDetector, ident ql.Ident) (ql.Ident, error)
CastModuleFieldToColumnType func(ModuleFieldTypeDetector, string) (string, error)
}
)
store/sqlite/sql_typecasters.go
+6 -7
View File
@@ -2,21 +2,20 @@ package sqlite
import (
"fmt"
"github.com/cortezaproject/corteza-server/pkg/ql"
"github.com/cortezaproject/corteza-server/store/rdbms"
)
func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, i ql.Ident) (ql.Ident, error) {
func fieldToColumnTypeCaster(field rdbms.ModuleFieldTypeDetector, ident string) (string, error) {
switch true {
case field.IsBoolean():
i.Value = fmt.Sprintf("(rv_%s.value NOT IN ('', '0', 'false', 'f', 'FALSE', 'F', false))", i.Value)
return fmt.Sprintf("(rv_%s.value NOT IN ('', '0', 'false', 'f', 'FALSE', 'F', false))", ident), nil
case field.IsNumeric():
i.Value = fmt.Sprintf("CAST(rv_%s.value AS SIGNED)", i.Value)
return fmt.Sprintf("CAST(rv_%s.value AS SIGNED)", ident), nil
case field.IsRef():
i.Value = fmt.Sprintf("rv_%s.ref ", i.Value)
return fmt.Sprintf("rv_%s.ref ", ident), nil
default:
i.Value = fmt.Sprintf("rv_%s.value ", i.Value)
return fmt.Sprintf("rv_%s.value ", ident), nil
}
return i, nil
return ident, nil
}
store/sqlite/sqlite.go
+4
View File
@@ -18,6 +18,10 @@ type (
}
)
func init() {
store.Register(Connect, "sqlite3", "sqlite")
}
func Connect(ctx context.Context, dsn string) (store.Storer, error) {
var (
err error
store/tests/compose_records_test.go
+24 -1
View File
@@ -415,7 +415,30 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) {
req.NoError(err)
// Note that not all functions are compatible across all backends
})
t.Run("partial value update", func(t *testing.T) {
var (
err error
set types.RecordSet
req, _ = truncAndCreate(t,
makeNew(&types.RecordValue{Name: "str1", Value: "1st"}, &types.RecordValue{Name: "num1", Value: "1"}),
makeNew(&types.RecordValue{Name: "str1", Value: "2nd"}, &types.RecordValue{Name: "num1", Value: "2"}),
makeNew(&types.RecordValue{Name: "str1", Value: "3rd"}, &types.RecordValue{Name: "num1", Value: "3"}),
)
)
set, _, err = s.SearchComposeRecords(ctx, mod, types.RecordFilter{})
req.NoError(err)
req.Equal("1st,1;2nd,2;3rd,3", stringifyValues(set, "str1", "num1"))
ups := &types.RecordValue{RecordID: set[1].ID, Name: "num1", Value: "22"}
req.NoError(s.PartialComposeRecordValueUpdate(ctx, mod, ups))
set, _, err = s.SearchComposeRecords(ctx, mod, types.RecordFilter{})
req.NoError(err)
req.Equal("1st,1;2nd,22;3rd,3", stringifyValues(set, "str1", "num1"))
})
}
tests/compose/module_test.go
+7 -1
View File
@@ -21,7 +21,12 @@ func (h helper) clearModules() {
}
func (h helper) makeModule(ns *types.Namespace, name string, ff ...*types.ModuleField) *types.Module {
return h.createModule(&types.Module{Name: name, NamespaceID: ns.ID, Fields: ff})
return h.createModule(&types.Module{
Name: name,
NamespaceID: ns.ID,
Fields: ff,
CreatedAt: time.Now(),
})
}
func (h helper) createModule(res *types.Module) *types.Module {
@@ -32,6 +37,7 @@ func (h helper) createModule(res *types.Module) *types.Module {
_ = res.Fields.Walk(func(f *types.ModuleField) error {
f.ID = id.Next()
f.ModuleID = res.ID
f.CreatedAt = time.Now()
return nil
})
tests/compose/record_exec_test.go
+36 -13
View File
@@ -7,6 +7,7 @@ import (
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/filter"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/tests/helpers"
"github.com/steinfletcher/apitest"
"net/http"
@@ -35,11 +36,9 @@ func TestRecordExecUnknownProcedure(t *testing.T) {
End()
}
func TestRecordExec(t *testing.T) {
t.Skipf("disabled due to unconsistent test results in build pipeline")
return
func TestRecordExecOrganize(t *testing.T) {
h := newHelper(t)
h.clearRecords()
h.allow(types.ModulePermissionResource.AppendWildcard(), "record.update")
@@ -61,7 +60,7 @@ func TestRecordExec(t *testing.T) {
assertSort := func(expectedHandles, expectedCats string) {
// Using record service for fetching to avoid value pre-fetching etc..
sorting, _ := filter.NewSorting("position ASC")
set, _, err := service.DefaultRecord.With(h.secCtx()).Find(types.RecordFilter{
set, _, err := store.SearchComposeRecords(h.secCtx(), service.DefaultNgStore, module, types.RecordFilter{
ModuleID: module.ID,
NamespaceID: module.NamespaceID,
Sorting: sorting,
@@ -74,6 +73,12 @@ func TestRecordExec(t *testing.T) {
actualCats := ""
_ = set.Walk(func(r *types.Record) error {
//fmt.Printf("%d\t%s\t%s\t%s\n", r.ID,
// r.Values.FilterByName("position")[0].Value,
// r.Values.FilterByName("handle")[0].Value,
// r.Values.FilterByName("category")[0].Value,
//)
v := r.Values.FilterByName("handle")
if len(v) == 1 {
@@ -91,6 +96,8 @@ func TestRecordExec(t *testing.T) {
h.a.Equal(expectedCats, actualCats)
}
t.Logf("seeding records")
var (
aRec = makeRecord(1, "a", "CAT1")
bRec = makeRecord(2, "b", "CAT1")
@@ -116,13 +123,18 @@ func TestRecordExec(t *testing.T) {
"i": strconv.FormatUint(iRec.ID, 10),
}
t.Logf("testing initial order")
assertSort("abcdefghi", "111222333")
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
t.Logf("moving 'a' to position '6'")
// Move a to the middle
h.apiSendRecordExec(module.NamespaceID, module.ID, "organize", request.ProcedureArgs{
{"recordID", rr["a"]},
{"positionField", "position"},
{"position", "5"}}).
{"position", "6"}}).
Status(http.StatusOK).
Assert(helpers.AssertNoErrors).
End()
@@ -131,6 +143,10 @@ func TestRecordExec(t *testing.T) {
// ^---v
assertSort("bcdeafghi", "112212333")
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
t.Logf("moving 'i' to position '0'")
// Move i to the beginning
h.apiSendRecordExec(module.NamespaceID, module.ID, "organize", request.ProcedureArgs{
{"recordID", rr["i"]},
@@ -144,6 +160,10 @@ func TestRecordExec(t *testing.T) {
// v<------^
assertSort("ibcdeafgh", "311221233")
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
t.Logf("moving 'b' to position '5'")
// Move b to the 5th place
h.apiSendRecordExec(module.NamespaceID, module.ID, "organize", request.ProcedureArgs{
{"recordID", rr["b"]},
@@ -156,26 +176,29 @@ func TestRecordExec(t *testing.T) {
// ibcdeafgh
// ^->v
assertSort("icdebfagh", "312212133")
assertSort("icdebafgh", "312211233")
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
t.Logf("moving 'b' to category CAT2")
// This will keep order of letters but move b to category-2
h.apiSendRecordExec(module.NamespaceID, module.ID, "organize", request.ProcedureArgs{
{"recordID", rr["b"]},
{"groupField", "category"},
{"group", "CAT2"},
{"position", "5"}}).
{"group", "CAT2"}}).
Status(http.StatusOK).
Assert(helpers.AssertNoErrors).
End()
// icdebfagh
// ^
assertSort("icdebfagh", "312222133")
assertSort("icdebafgh", "312221233")
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
// ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
lRec := h.lookupRecordByID(module, bRec.ID)
//rsv, err := h.repoRecord().LoadValues([]string{"category"}, []uint64{bRec.ID})
//h.noError(err)
h.a.NotNil(lRec.Values)
h.a.Len(lRec.Values.FilterByName("category"), 1)
h.a.Equal("CAT2", lRec.Values.FilterByName("category")[0].Value)
tests/helpers/app.go
+4 -7
View File
@@ -6,20 +6,17 @@ import (
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/pkg/rand"
"github.com/cortezaproject/corteza-server/store/sqlite"
"time"
// Explicitly register SQLite (not done in the app as for testing only)
_ "github.com/cortezaproject/corteza-server/store/sqlite"
)
func NewIntegrationTestApp(ctx context.Context, initTestServices func(*app.CortezaApp) error) *app.CortezaApp {
var (
a = app.New()
err error
a = app.New()
)
// Make sure SQLite is registered
a.Store, err = sqlite.ConnectInMemory(ctx)
cli.HandleError(err)
// When running integration tests, we want to upgrade the db. Always.
a.Opt.Upgrade.Always = true