DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Fix sorting&paging on RDBMS store

Browse Source
This commit is contained in:
Denis Arh
2020-11-12 19:21:33 +01:00
parent 82fd6309e0
commit b7de337db9
42 changed files with 1206 additions and 1050 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/types/attachment.go
+1 -2
View File
@@ -36,8 +36,6 @@ type (
FieldName string `json:"fieldName,omitempty"`
Filter string `json:"filter"`
Sort string `json:"sort"`
// Check fn is called by store backend for each resource found function can
// modify the resource and return false if store should not return it
//
@@ -45,6 +43,7 @@ type (
Check func(*Attachment) (bool, error)
// Standard helpers for paging and sorting
filter.Sorting
filter.Paging
}
pkg/codegen/assets/store_rdbms.gen.go.tpl
+97 -109
View File
@@ -42,6 +42,10 @@ const (
*/}}
{{ if $.Search.Enable }}
{{ if $.Search.Custom }}
// {{ toggleExport .Search.Export "Search" $.Types.Plural }} not generated
// {search: {custom:true}}
{{ else }}
// {{ toggleExport .Search.Export "Search" $.Types.Plural }} returns all matching rows
//
// This function calls convert{{ export $.Types.Singular }}Filter with the given
@@ -53,83 +57,78 @@ func (s Store) {{ toggleExport .Search.Export "Search" $.Types.Plural }}(ctx con
q squirrel.SelectBuilder
)
{{- if .RDBMS.CustomFilterConverter }}
q, err = s.convert{{ export $.Types.Singular }}Filter({{ template "extraArgsCallFirst" . }}f)
if err != nil {
return nil, f, err
}
{{- else }}
q = s.{{ unexport $.Types.Plural }}SelectBuilder()
{{- end }}
{{ if $.Search.EnablePaging }}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
{{ end }}
{{ if and $.Search.EnableSorting $.Search.EnablePaging }}
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
{{ else if $.Search.EnablePaging }}
// Sorting is disabled in definition yaml file
// {search: {enableSorting:false}}
//
// We still need to sort the results by primary key for paging purposes
curSort := filter.SortExprSet{
{{- range $.RDBMS.Columns.PrimaryKeyFields }}
&filter.SortExpr{Column: {{ printf "%q" .Column }}, {{ if .SortDescending }}Descending: !reversedCursor, {{ end }}},
{{- end }}
}
{{ end }}
return set, f, func() error {
{{- if $.Search.EnablePaging -}}
set, err = s.{{ unexport "fetchFullPageOf" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, curSort, f.PageCursor, f.Limit, {{ if $.Search.EnableFilterCheckFn }}f.Check{{ else }}nil{{ end }},)
{{- if .RDBMS.CustomFilterConverter }}
q, err = s.convert{{ export $.Types.Singular }}Filter({{ template "extraArgsCallFirst" . }}f)
if err != nil {
return err
}
{{- else }}
q = s.{{ unexport $.Types.Plural }}SelectBuilder()
{{- end }}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collect{{ export $.Types.Singular }}CursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
{{ if $.Search.EnablePaging }}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collect{{ export $.Types.Singular }}CursorValues(set[len(set)-1], curSort.Columns()...)
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
{{- range $.RDBMS.Columns.PrimaryKeyFields }}
if f.Sort.Get({{ printf "%q" .Column }}) == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: {{ printf "%q" .Column }}, {{ if .SortDescending }}Descending: true, {{ end }}})
}
{{- end }}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
{{ end }}
{{ if .RDBMS.CustomSortConverter }}
if q, err = s.{{ unexport $.Types.Plural }}Sorter({{ template "extraArgsCallFirst" . }}q, f.Sort); err != nil {
return err
}
{{ else if $.Search.EnableSorting }}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortable{{ export $.Types.Singular }}Columns()); err != nil {
return err
}
{{ end }}
{{- if $.Search.EnablePaging }}
set, f.PrevPage, f.NextPage, err = s.{{ unexport "fetchFullPageOf" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, {{ if $.Search.EnableFilterCheckFn }}f.Check{{ else }}nil{{ end }},)
if err != nil {
return err
}
f.PageCursor = nil
return nil
{{- else }}
{{- if $.Search.EnableSorting }}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, f.Sort, s.sortable{{ export $.Types.Singular }}Columns()); err != nil {
return err
}
{{ end -}}
set, _, _, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, {{ if $.Search.EnableFilterCheckFn }}f.Check{{else}}nil{{ end }})
return err
{{ end }}
{{- end }}
}()
}
{{ end }}
{{ end }}
{{ if $.Search.EnablePaging }}
@@ -137,75 +136,65 @@ func (s Store) {{ toggleExport .Search.Export "Search" $.Types.Plural }}(ctx con
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural }} (
ctx context.Context{{ template "extraArgsDef" . }},
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*{{ $.Types.GoType }}) (bool, error),
) ([]*{{ $.Types.GoType }}, error) {
) (set []*{{ $.Types.GoType }}, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*{{ $.Types.GoType }}, 0, DefaultSliceCapacity)
aux []*{{ $.Types.GoType }}
last *{{ $.Types.GoType }}
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
set = make([]*{{ $.Types.GoType }}, 0, DefaultSliceCapacity)
{{ if .RDBMS.Columns.PrimaryKeyFields }}
// Make sure we always end our sort by primary keys
{{- range .RDBMS.Columns.PrimaryKeyFields }}
if sort.Get({{ printf "%q" .Column }}) == nil {
sort = append(sort, &filter.SortExpr{Column: {{ printf "%q" .Column }}})
if cursor != nil {
cursor.Reverse = sortDesc
}
{{ end }}
{{ end }}
{{ if .RDBMS.CustomSortConverter }}
if q, err = s.{{ unexport $.Types.Plural }}Sorter({{ template "extraArgsCallFirst" . }}q, sort); err != nil {
return nil, err
}
{{ else if .Search.EnableSorting }}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortable{{ export $.Types.Singular }}Columns()); err != nil {
return nil, err
}
{{ else if .RDBMS.Columns.PrimaryKeyFields }}
// Sort by primary keys by default
if q, err = setOrderBy(q, sort, nil); err != nil {
return nil, err
}
{{ end }}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collect{{ export $.Types.Singular }}CursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collect{{ export $.Types.Singular }}CursorValues(aux[limit-1], sortColumns...);
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -224,23 +213,23 @@ func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural }} (
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collect{{ export $.Types.Singular }}CursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
{{ end }}
@@ -281,13 +270,12 @@ func (s Store) {{ export "query" $.Types.Plural }} (
}
{{ if $.Search.EnableFilterCheckFn }}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
pkg/codegen/store.go
+2
View File
@@ -175,6 +175,7 @@ type (
storeTypeSearchDef struct {
Enable bool `yaml:"enable"`
Export bool `yaml:"export"`
Custom bool `yaml:"custom"`
EnablePaging bool `yaml:"enablePaging"`
EnableSorting bool `yaml:"enableSorting"`
EnableFilterCheckFn bool `yaml:"enableFilterCheckFunction"`
@@ -225,6 +226,7 @@ func procStore(mm ...string) ([]*storeDef, error) {
Search: storeTypeSearchDef{
Enable: true,
Export: true,
Custom: false,
EnablePaging: true,
EnableSorting: true,
EnableFilterCheckFn: true,
pkg/filter/pagination.go
+41 -1
View File
@@ -77,7 +77,11 @@ func (p *PagingCursor) String() string {
var o = "<"
for i, key := range p.keys {
o += fmt.Sprintf("%s: %v, ", key, p.values[i])
o += fmt.Sprintf("%s: %v", key, p.values[i])
if p.desc[i] {
o += " DESC"
}
o += ", "
}
if p.Reverse {
@@ -160,6 +164,42 @@ func (p *PagingCursor) Decode(cursor string) error {
return err
}
// Sort returns:
// - sort if cursor is nil
// - sort from cursor when sort is empty
// - sort from cursor when sort is compatible with cursor
// - error if sort & cursor are incompatible
func (p *PagingCursor) Sort(sort SortExprSet) (SortExprSet, error) {
if p == nil {
return sort, nil
}
if len(sort) == 0 {
// sort emprt, create it from cursor
for k := range p.keys {
sort = append(sort, &SortExpr{
Column: p.keys[k],
Descending: p.desc[k],
})
}
return sort, nil
}
// check compatibility
ss := sort.Columns()
if len(p.keys) != len(ss) {
return nil, fmt.Errorf("incompatible sort")
}
for k := range p.keys {
if p.keys[k] != ss[k] {
return nil, fmt.Errorf("incompatible sort")
}
}
return sort, nil
}
func parseCursor(in string) (p *PagingCursor, err error) {
if len(in) == 0 {
return nil, nil
pkg/filter/pagination_test.go
+3 -3
View File
@@ -19,10 +19,10 @@ func Test_cursorEncDec(t *testing.T) {
{
cur.Set("uint64", id, true)
cur.Set("string", "foo", true)
cur.Set("string", "foo", false)
req.Len(cur.values, 2)
req.Equal(id, cur.values[0])
req.Equal(fmt.Sprintf("<uint64: %d, string: foo, forward>", id), cur.String())
req.Equal(fmt.Sprintf("<uint64: %d DESC, string: foo, forward>", id), cur.String())
}
{
@@ -34,7 +34,7 @@ func Test_cursorEncDec(t *testing.T) {
req.Len(dec.values, 2)
req.Equal(id, dec.values[0])
req.Equal("foo", dec.values[1])
req.Equal(fmt.Sprintf("<uint64: %d, string: foo, forward>", id), cur.String())
req.Equal(fmt.Sprintf("<uint64: %d DESC, string: foo, forward>", id), cur.String())
}
}
pkg/filter/sorting.go
+9
View File
@@ -125,6 +125,15 @@ func (set SortExprSet) Reverse() {
}
}
// Sorting is revered if 1st expr has desc direction
func (set SortExprSet) Reversed() bool {
if len(set) > 0 {
return set[0].Descending
}
return false
}
// Reverse reverses direction on each expression
func (set SortExprSet) Columns() []string {
out := make([]string, len(set))
store/actionlog.yaml
+1 -2
View File
@@ -8,7 +8,7 @@ types:
filterType: actionlog.Filter
fields:
- { field: ID, sortDescending: true }
- { field: ID, sortable: true, sortDescending: true }
- { field: Timestamp, type: "time.Time" }
- { field: RequestOrigin }
- { field: RequestID }
@@ -33,7 +33,6 @@ rdbms:
ActorID: { column: actor_id }
search:
enableSorting: false
enableFilterCheckFunction: false
update:
store/compose_records.yaml
+1
View File
@@ -53,6 +53,7 @@ rdbms:
ModuleID: { column: module_id }
search:
custom: true
export: false
create:
store/rdbms/actionlog.gen.go
+83 -60
View File
@@ -30,42 +30,50 @@ func (s Store) SearchActionlogs(ctx context.Context, f actionlog.Filter) (action
set []*actionlog.Action
q squirrel.SelectBuilder
)
q, err = s.convertActionlogFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting is disabled in definition yaml file
// {search: {enableSorting:false}}
//
// We still need to sort the results by primary key for paging purposes
curSort := filter.SortExprSet{
&filter.SortExpr{Column: "id", Descending: !reversedCursor},
}
return set, f, func() error {
set, err = s.fetchFullPageOfActionlogs(ctx, q, curSort, f.PageCursor, f.Limit, nil)
q, err = s.convertActionlogFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectActionlogCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectActionlogCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id", Descending: true})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableActionlogColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfActionlogs(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, nil)
if err != nil {
return err
}
f.PageCursor = nil
@@ -77,58 +85,64 @@ func (s Store) SearchActionlogs(ctx context.Context, f actionlog.Filter) (action
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfActionlogs(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*actionlog.Action) (bool, error),
) ([]*actionlog.Action, error) {
) (set []*actionlog.Action, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*actionlog.Action, 0, DefaultSliceCapacity)
aux []*actionlog.Action
last *actionlog.Action
aux []*actionlog.Action
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*actionlog.Action, 0, DefaultSliceCapacity)
// Sort by primary keys by default
if q, err = setOrderBy(q, sort, nil); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryActionlogs(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryActionlogs(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectActionlogCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectActionlogCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -147,23 +161,23 @@ func (s Store) fetchFullPageOfActionlogs(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectActionlogCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryActionlogs queries the database, converts and checks each row and
@@ -315,7 +329,16 @@ func (Store) actionlogColumns(aa ...string) []string {
}
}
// {true true true false false}
// {true true false true true false}
// sortableActionlogColumns returns all Actionlog columns flagged as sortable
//
// With optional string arg, all columns are returned aliased
func (Store) sortableActionlogColumns() map[string]string {
return map[string]string{
"id": "id",
}
}
// internalActionlogEncoder encodes fields from actionlog.Action to store.Payload (map)
//
store/rdbms/applications.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchApplications(ctx context.Context, f types.ApplicationFilter
set []*types.Application
q squirrel.SelectBuilder
)
q, err = s.convertApplicationFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfApplications(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertApplicationFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectApplicationCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectApplicationCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableApplicationColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfApplications(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchApplications(ctx context.Context, f types.ApplicationFilter
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfApplications(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Application) (bool, error),
) ([]*types.Application, error) {
) (set []*types.Application, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Application, 0, DefaultSliceCapacity)
aux []*types.Application
last *types.Application
aux []*types.Application
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Application, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableApplicationColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryApplications(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryApplications(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectApplicationCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectApplicationCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfApplications(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectApplicationCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryApplications queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryApplications(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -434,7 +445,7 @@ func (Store) applicationColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableApplicationColumns returns all Application columns flagged as sortable
//
store/rdbms/attachments.gen.go
+8 -9
View File
@@ -29,15 +29,15 @@ func (s Store) SearchAttachments(ctx context.Context, f types.AttachmentFilter)
set []*types.Attachment
q squirrel.SelectBuilder
)
q, err = s.convertAttachmentFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertAttachmentFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryAttachments(ctx, q, f.Check)
return err
}()
}
@@ -76,13 +76,12 @@ func (s Store) QueryAttachments(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -311,7 +310,7 @@ func (Store) attachmentColumns(aa ...string) []string {
}
}
// {true true false false true}
// {true true false false false true}
// internalAttachmentEncoder encodes fields from types.Attachment to store.Payload (map)
//
store/rdbms/compose_attachments.gen.go
+8 -9
View File
@@ -29,15 +29,15 @@ func (s Store) SearchComposeAttachments(ctx context.Context, f types.AttachmentF
set []*types.Attachment
q squirrel.SelectBuilder
)
q, err = s.convertComposeAttachmentFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertComposeAttachmentFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryComposeAttachments(ctx, q, f.Check)
return err
}()
}
@@ -76,13 +76,12 @@ func (s Store) QueryComposeAttachments(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -313,7 +312,7 @@ func (Store) composeAttachmentColumns(aa ...string) []string {
}
}
// {true true false false true}
// {true true false false false true}
// internalComposeAttachmentEncoder encodes fields from types.Attachment to store.Payload (map)
//
store/rdbms/compose_charts.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchComposeCharts(ctx context.Context, f types.ChartFilter) (ty
set []*types.Chart
q squirrel.SelectBuilder
)
q, err = s.convertComposeChartFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfComposeCharts(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertComposeChartFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectComposeChartCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectComposeChartCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposeChartColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeCharts(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchComposeCharts(ctx context.Context, f types.ChartFilter) (ty
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfComposeCharts(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Chart) (bool, error),
) ([]*types.Chart, error) {
) (set []*types.Chart, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Chart, 0, DefaultSliceCapacity)
aux []*types.Chart
last *types.Chart
aux []*types.Chart
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Chart, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposeChartColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryComposeCharts(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryComposeCharts(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectComposeChartCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectComposeChartCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposeCharts(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectComposeChartCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryComposeCharts queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryComposeCharts(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -444,7 +455,7 @@ func (Store) composeChartColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableComposeChartColumns returns all ComposeChart columns flagged as sortable
//
store/rdbms/compose_module_fields.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchComposeModuleFields(ctx context.Context, f types.ModuleFiel
set []*types.ModuleField
q squirrel.SelectBuilder
)
q, err = s.convertComposeModuleFieldFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertComposeModuleFieldFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryComposeModuleFields(ctx, q, nil)
return err
}()
}
@@ -311,7 +311,7 @@ func (Store) composeModuleFieldColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalComposeModuleFieldEncoder encodes fields from types.ModuleField to store.Payload (map)
//
store/rdbms/compose_modules.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchComposeModules(ctx context.Context, f types.ModuleFilter) (
set []*types.Module
q squirrel.SelectBuilder
)
q, err = s.convertComposeModuleFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfComposeModules(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertComposeModuleFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectComposeModuleCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectComposeModuleCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposeModuleColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeModules(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchComposeModules(ctx context.Context, f types.ModuleFilter) (
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfComposeModules(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Module) (bool, error),
) ([]*types.Module, error) {
) (set []*types.Module, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Module, 0, DefaultSliceCapacity)
aux []*types.Module
last *types.Module
aux []*types.Module
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Module, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposeModuleColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryComposeModules(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryComposeModules(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectComposeModuleCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectComposeModuleCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposeModules(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectComposeModuleCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryComposeModules queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryComposeModules(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -454,7 +465,7 @@ func (Store) composeModuleColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableComposeModuleColumns returns all ComposeModule columns flagged as sortable
//
store/rdbms/compose_namespaces.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchComposeNamespaces(ctx context.Context, f types.NamespaceFil
set []*types.Namespace
q squirrel.SelectBuilder
)
q, err = s.convertComposeNamespaceFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfComposeNamespaces(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertComposeNamespaceFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectComposeNamespaceCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectComposeNamespaceCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposeNamespaceColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeNamespaces(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchComposeNamespaces(ctx context.Context, f types.NamespaceFil
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfComposeNamespaces(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Namespace) (bool, error),
) ([]*types.Namespace, error) {
) (set []*types.Namespace, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Namespace, 0, DefaultSliceCapacity)
aux []*types.Namespace
last *types.Namespace
aux []*types.Namespace
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Namespace, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposeNamespaceColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryComposeNamespaces(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryComposeNamespaces(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectComposeNamespaceCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectComposeNamespaceCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposeNamespaces(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectComposeNamespaceCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryComposeNamespaces queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryComposeNamespaces(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -443,7 +454,7 @@ func (Store) composeNamespaceColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableComposeNamespaceColumns returns all ComposeNamespace columns flagged as sortable
//
store/rdbms/compose_pages.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchComposePages(ctx context.Context, f types.PageFilter) (type
set []*types.Page
q squirrel.SelectBuilder
)
q, err = s.convertComposePageFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfComposePages(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertComposePageFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectComposePageCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectComposePageCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposePageColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposePages(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchComposePages(ctx context.Context, f types.PageFilter) (type
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfComposePages(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Page) (bool, error),
) ([]*types.Page, error) {
) (set []*types.Page, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Page, 0, DefaultSliceCapacity)
aux []*types.Page
last *types.Page
aux []*types.Page
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Page, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableComposePageColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryComposePages(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryComposePages(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectComposePageCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectComposePageCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposePages(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectComposePageCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryComposePages queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryComposePages(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -464,7 +475,7 @@ func (Store) composePageColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableComposePageColumns returns all ComposePage columns flagged as sortable
//
store/rdbms/compose_record_values.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) searchComposeRecordValues(ctx context.Context, _mod *types.Module
set []*types.RecordValue
q squirrel.SelectBuilder
)
q, err = s.convertComposeRecordValueFilter(_mod, f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertComposeRecordValueFilter(_mod, f)
if err != nil {
return err
}
set, _, _, err = s.QueryComposeRecordValues(ctx, _mod, q, nil)
return err
}()
}
@@ -287,7 +287,7 @@ func (Store) composeRecordValueColumns(aa ...string) []string {
}
}
// {true false false false false}
// {true false false false false false}
// internalComposeRecordValueEncoder encodes fields from types.RecordValue to store.Payload (map)
//
store/rdbms/compose_records.gen.go
+41 -87
View File
@@ -20,116 +20,71 @@ import (
var _ = errors.Is
// searchComposeRecords returns all matching rows
//
// This function calls convertComposeRecordFilter with the given
// types.RecordFilter and expects to receive a working squirrel.SelectBuilder
func (s Store) searchComposeRecords(ctx context.Context, _mod *types.Module, f types.RecordFilter) (types.RecordSet, types.RecordFilter, error) {
var (
err error
set []*types.Record
q squirrel.SelectBuilder
)
q, err = s.convertComposeRecordFilter(_mod, f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfComposeRecords(ctx, _mod, q, curSort, f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectComposeRecordCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectComposeRecordCursorValues(set[len(set)-1], curSort.Columns()...)
}
f.PageCursor = nil
return nil
}()
}
// searchComposeRecords not generated
// {search: {custom:true}}
// fetchFullPageOfComposeRecords collects all requested results.
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfComposeRecords(
ctx context.Context, _mod *types.Module,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Record) (bool, error),
) ([]*types.Record, error) {
) (set []*types.Record, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Record, 0, DefaultSliceCapacity)
aux []*types.Record
last *types.Record
aux []*types.Record
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Record, 0, DefaultSliceCapacity)
if q, err = s.composeRecordsSorter(_mod, q, sort); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryComposeRecords(ctx, _mod, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryComposeRecords(ctx, _mod, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectComposeRecordCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectComposeRecordCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -148,23 +103,23 @@ func (s Store) fetchFullPageOfComposeRecords(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectComposeRecordCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryComposeRecords queries the database, converts and checks each row and
@@ -202,13 +157,12 @@ func (s Store) QueryComposeRecords(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -444,7 +398,7 @@ func (Store) composeRecordColumns(aa ...string) []string {
}
}
// {true false true true true}
// {true false true true true true}
// sortableComposeRecordColumns returns all ComposeRecord columns flagged as sortable
//
store/rdbms/compose_records.go
+35 -9
View File
@@ -22,14 +22,40 @@ const (
// physical columns are part of module-field configuration
// SearchComposeRecords returns all matching ComposeRecords from store
func (s Store) SearchComposeRecords(ctx context.Context, m *types.Module, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) {
// In when module requires this,
set, f, err = s.searchComposeRecords(ctx, m, filter)
if err != nil {
return
}
func (s Store) SearchComposeRecords(ctx context.Context, m *types.Module, f types.RecordFilter) (types.RecordSet, types.RecordFilter, error) {
var (
err error
set []*types.Record
q squirrel.SelectBuilder
)
return
return set, f, func() error {
q, err = s.convertComposeRecordFilter(m, f)
if err != nil {
return err
}
f.PrevPage, f.NextPage = nil, nil
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
if q, err = s.composeRecordsSorter(m, q, f.Sort); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeRecords(ctx, m, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
return nil
}()
}
// LookupComposeRecordByID searches for compose record by ID
@@ -351,7 +377,7 @@ func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string)
collect = func(cc ...string) {
for _, c := range cc {
switch c {
case "crd.id":
case "id":
cursor.Set(c, res.ID, false)
pkID = true
case "crd.created_at":
@@ -374,7 +400,7 @@ func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string)
collect(cc...)
if !hasUnique || !pkID {
collect(
"crd.id",
"id",
)
}
store/rdbms/credentials.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchCredentials(ctx context.Context, f types.CredentialsFilter)
set []*types.Credentials
q squirrel.SelectBuilder
)
q, err = s.convertCredentialsFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertCredentialsFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryCredentials(ctx, q, nil)
return err
}()
}
@@ -302,7 +302,7 @@ func (Store) credentialsColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalCredentialsEncoder encodes fields from types.Credentials to store.Payload (map)
//
store/rdbms/labels.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchLabels(ctx context.Context, f types.LabelFilter) (types.Lab
set []*types.Label
q squirrel.SelectBuilder
)
q, err = s.convertLabelFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertLabelFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryLabels(ctx, q, nil)
return err
}()
}
@@ -292,7 +292,7 @@ func (Store) labelColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalLabelEncoder encodes fields from types.Label to store.Payload (map)
//
store/rdbms/messaging_attachments.gen.go
+8 -9
View File
@@ -29,15 +29,15 @@ func (s Store) SearchMessagingAttachments(ctx context.Context, f types.Attachmen
set []*types.Attachment
q squirrel.SelectBuilder
)
q, err = s.convertMessagingAttachmentFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertMessagingAttachmentFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryMessagingAttachments(ctx, q, f.Check)
return err
}()
}
@@ -76,13 +76,12 @@ func (s Store) QueryMessagingAttachments(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -309,7 +308,7 @@ func (Store) messagingAttachmentColumns(aa ...string) []string {
}
}
// {true true false false true}
// {true true false false false true}
// internalMessagingAttachmentEncoder encodes fields from types.Attachment to store.Payload (map)
//
store/rdbms/messaging_channel_members.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchMessagingChannelMembers(ctx context.Context, f types.Channe
set []*types.ChannelMember
q squirrel.SelectBuilder
)
q, err = s.convertMessagingChannelMemberFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertMessagingChannelMemberFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryMessagingChannelMembers(ctx, q, nil)
return err
}()
}
@@ -285,7 +285,7 @@ func (Store) messagingChannelMemberColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalMessagingChannelMemberEncoder encodes fields from types.ChannelMember to store.Payload (map)
//
store/rdbms/messaging_channels.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchMessagingChannels(ctx context.Context, f types.ChannelFilte
set []*types.Channel
q squirrel.SelectBuilder
)
q, err = s.convertMessagingChannelFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfMessagingChannels(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertMessagingChannelFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectMessagingChannelCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectMessagingChannelCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableMessagingChannelColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfMessagingChannels(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchMessagingChannels(ctx context.Context, f types.ChannelFilte
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfMessagingChannels(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Channel) (bool, error),
) ([]*types.Channel, error) {
) (set []*types.Channel, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Channel, 0, DefaultSliceCapacity)
aux []*types.Channel
last *types.Channel
aux []*types.Channel
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Channel, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableMessagingChannelColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryMessagingChannels(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryMessagingChannels(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectMessagingChannelCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectMessagingChannelCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfMessagingChannels(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectMessagingChannelCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryMessagingChannels queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryMessagingChannels(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -442,7 +453,7 @@ func (Store) messagingChannelColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableMessagingChannelColumns returns all MessagingChannel columns flagged as sortable
//
store/rdbms/messaging_flags.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchMessagingFlags(ctx context.Context, f types.MessageFlagFilt
set []*types.MessageFlag
q squirrel.SelectBuilder
)
q, err = s.convertMessagingFlagFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertMessagingFlagFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryMessagingFlags(ctx, q, nil)
return err
}()
}
@@ -290,7 +290,7 @@ func (Store) messagingFlagColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalMessagingFlagEncoder encodes fields from types.MessageFlag to store.Payload (map)
//
store/rdbms/messaging_mentions.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchMessagingMentions(ctx context.Context, f types.MentionFilte
set []*types.Mention
q squirrel.SelectBuilder
)
q, err = s.convertMessagingMentionFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertMessagingMentionFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryMessagingMentions(ctx, q, nil)
return err
}()
}
@@ -292,7 +292,7 @@ func (Store) messagingMentionColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalMessagingMentionEncoder encodes fields from types.Mention to store.Payload (map)
//
store/rdbms/messaging_message_attachments.gen.go
+1 -1
View File
@@ -260,7 +260,7 @@ func (Store) messagingMessageAttachmentColumns(aa ...string) []string {
}
}
// {false true false false false}
// {false true false false false false}
// internalMessagingMessageAttachmentEncoder encodes fields from types.MessageAttachment to store.Payload (map)
//
store/rdbms/messaging_messages.gen.go
+6 -6
View File
@@ -29,15 +29,15 @@ func (s Store) SearchMessagingMessages(ctx context.Context, f types.MessageFilte
set []*types.Message
q squirrel.SelectBuilder
)
q, err = s.convertMessagingMessageFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertMessagingMessageFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QueryMessagingMessages(ctx, q, nil)
return err
}()
}
@@ -302,7 +302,7 @@ func (Store) messagingMessageColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalMessagingMessageEncoder encodes fields from types.Message to store.Payload (map)
//
store/rdbms/messaging_unread.gen.go
+1 -1
View File
@@ -263,7 +263,7 @@ func (Store) messagingUnreadColumns(aa ...string) []string {
}
}
// {false true false false false}
// {false true false false false false}
// internalMessagingUnreadEncoder encodes fields from types.Unread to store.Payload (map)
//
store/rdbms/rbac_rules.gen.go
+3 -3
View File
@@ -29,12 +29,12 @@ func (s Store) SearchRbacRules(ctx context.Context, f rbac.RuleFilter) (rbac.Rul
set []*rbac.Rule
q squirrel.SelectBuilder
)
q = s.rbacRulesSelectBuilder()
return set, f, func() error {
q = s.rbacRulesSelectBuilder()
set, _, _, err = s.QueryRbacRules(ctx, q, nil)
return err
}()
}
@@ -280,7 +280,7 @@ func (Store) rbacRuleColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalRbacRuleEncoder encodes fields from rbac.Rule to store.Payload (map)
//
store/rdbms/rdbms.go
+3
View File
@@ -409,8 +409,11 @@ func tx(ctx context.Context, dbCandidate interface{}, cfg *Config, txOpt *sql.Tx
func setCursorCond(q squirrel.SelectBuilder, cursor *filter.PagingCursor) squirrel.SelectBuilder {
if cursor != nil && len(cursor.Keys()) > 0 {
const cursorTpl = `(%s) %s (?%s)`
//expecting row values to be greater then args
op := ">"
if cursor.Reverse {
//expecting row values to be lesser then args
op = "<"
}
store/rdbms/reminders.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchReminders(ctx context.Context, f types.ReminderFilter) (typ
set []*types.Reminder
q squirrel.SelectBuilder
)
q, err = s.convertReminderFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfReminders(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertReminderFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectReminderCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectReminderCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableReminderColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfReminders(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchReminders(ctx context.Context, f types.ReminderFilter) (typ
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfReminders(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Reminder) (bool, error),
) ([]*types.Reminder, error) {
) (set []*types.Reminder, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Reminder, 0, DefaultSliceCapacity)
aux []*types.Reminder
last *types.Reminder
aux []*types.Reminder
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Reminder, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableReminderColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryReminders(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryReminders(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectReminderCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectReminderCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfReminders(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectReminderCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryReminders queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryReminders(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -444,7 +455,7 @@ func (Store) reminderColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableReminderColumns returns all Reminder columns flagged as sortable
//
store/rdbms/role_members.gen.go
+3 -3
View File
@@ -29,12 +29,12 @@ func (s Store) SearchRoleMembers(ctx context.Context, f types.RoleMemberFilter)
set []*types.RoleMember
q squirrel.SelectBuilder
)
q = s.roleMembersSelectBuilder()
return set, f, func() error {
q = s.roleMembersSelectBuilder()
set, _, _, err = s.QueryRoleMembers(ctx, q, nil)
return err
}()
}
@@ -274,7 +274,7 @@ func (Store) roleMemberColumns(aa ...string) []string {
}
}
// {true true false false false}
// {true true false false false false}
// internalRoleMemberEncoder encodes fields from types.RoleMember to store.Payload (map)
//
store/rdbms/roles.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchRoles(ctx context.Context, f types.RoleFilter) (types.RoleS
set []*types.Role
q squirrel.SelectBuilder
)
q, err = s.convertRoleFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfRoles(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertRoleFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectRoleCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectRoleCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableRoleColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfRoles(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchRoles(ctx context.Context, f types.RoleFilter) (types.RoleS
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfRoles(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.Role) (bool, error),
) ([]*types.Role, error) {
) (set []*types.Role, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.Role, 0, DefaultSliceCapacity)
aux []*types.Role
last *types.Role
aux []*types.Role
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.Role, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableRoleColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryRoles(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryRoles(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectRoleCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectRoleCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfRoles(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectRoleCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryRoles queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryRoles(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -456,7 +467,7 @@ func (Store) roleColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableRoleColumns returns all Role columns flagged as sortable
//
store/rdbms/settings.gen.go
+8 -9
View File
@@ -29,15 +29,15 @@ func (s Store) SearchSettings(ctx context.Context, f types.SettingsFilter) (type
set []*types.SettingValue
q squirrel.SelectBuilder
)
q, err = s.convertSettingFilter(f)
if err != nil {
return nil, f, err
}
return set, f, func() error {
q, err = s.convertSettingFilter(f)
if err != nil {
return err
}
set, _, _, err = s.QuerySettings(ctx, q, f.Check)
return err
}()
}
@@ -76,13 +76,12 @@ func (s Store) QuerySettings(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -302,7 +301,7 @@ func (Store) settingColumns(aa ...string) []string {
}
}
// {true true false false true}
// {true true false false false true}
// internalSettingEncoder encodes fields from types.SettingValue to store.Payload (map)
//
store/rdbms/users.gen.go
+76 -65
View File
@@ -30,44 +30,50 @@ func (s Store) SearchUsers(ctx context.Context, f types.UserFilter) (types.UserS
set []*types.User
q squirrel.SelectBuilder
)
q, err = s.convertUserFilter(f)
if err != nil {
return nil, f, err
}
// Cleanup anything we've accidentally received...
f.PrevPage, f.NextPage = nil, nil
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse
// Sorting and paging are both enabled in definition yaml file
// {search: {enableSorting:true, enablePaging:true}}
curSort := f.Sort.Clone()
// If paging with reverse cursor, change the sorting
// direction for all columns we're sorting by
if reversedCursor {
curSort.Reverse()
}
return set, f, func() error {
set, err = s.fetchFullPageOfUsers(ctx, q, curSort, f.PageCursor, f.Limit, f.Check)
q, err = s.convertUserFilter(f)
if err != nil {
return err
}
if f.Limit > 0 && len(set) > 0 {
if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) {
f.PrevPage = s.collectUserCursorValues(set[0], curSort.Columns()...)
f.PrevPage.Reverse = true
}
// Paging enabled
// {search: {enablePaging:true}}
// Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned)
f.PrevPage, f.NextPage = nil, nil
// Less items fetched then requested by page-limit
// not very likely there's another page
f.NextPage = s.collectUserCursorValues(set[len(set)-1], curSort.Columns()...)
if f.PageCursor != nil {
// Page cursor exists so we need to validate it against used sort
if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
return err
}
}
if len(f.Sort) == 0 {
f.Sort = filter.SortExprSet{}
}
// Make sure results are always sorted at least by primary keys
if f.Sort.Get("id") == nil {
f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"})
}
sort := f.Sort.Clone()
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
if f.PageCursor != nil && f.PageCursor.Reverse {
sort.Reverse()
}
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableUserColumns()); err != nil {
return err
}
set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfUsers(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check)
if err != nil {
return err
}
f.PageCursor = nil
@@ -79,58 +85,64 @@ func (s Store) SearchUsers(ctx context.Context, f types.UserFilter) (types.UserS
//
// Function applies:
// - cursor conditions (where ...)
// - sorting rules (order by ...)
// - limit
//
// Main responsibility of this function is to perform additional sequential queries in case when not enough results
// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched
// are collected due to failed check on a specific row (by check fn).
//
// Function then moves cursor to the last item fetched
func (s Store) fetchFullPageOfUsers(
ctx context.Context,
q squirrel.SelectBuilder,
sort filter.SortExprSet,
sortColumns []string,
sortDesc bool,
cursor *filter.PagingCursor,
limit uint,
check func(*types.User) (bool, error),
) ([]*types.User, error) {
) (set []*types.User, prev, next *filter.PagingCursor, err error) {
var (
set = make([]*types.User, 0, DefaultSliceCapacity)
aux []*types.User
last *types.User
aux []*types.User
// When cursor for a previous page is used it's marked as reversed
// This tells us to flip the descending flag on all used sort keys
reversedCursor = cursor != nil && cursor.Reverse
reversedOrder = cursor != nil && cursor.Reverse
// copy of the select builder
tryQuery squirrel.SelectBuilder
fetched uint
err error
)
// Make sure we always end our sort by primary keys
if sort.Get("id") == nil {
sort = append(sort, &filter.SortExpr{Column: "id"})
}
set = make([]*types.User, 0, DefaultSliceCapacity)
// Apply sorting expr from filter to query
if q, err = setOrderBy(q, sort, s.sortableUserColumns()); err != nil {
return nil, err
if cursor != nil {
cursor.Reverse = sortDesc
}
for try := 0; try < MaxRefetches; try++ {
tryQuery = setCursorCond(q, cursor)
if limit > 0 {
tryQuery = tryQuery.Limit(uint64(limit))
tryQuery = tryQuery.Limit(uint64(limit + 1))
}
if aux, fetched, last, err = s.QueryUsers(ctx, tryQuery, check); err != nil {
return nil, err
if aux, fetched, _, err = s.QueryUsers(ctx, tryQuery, check); err != nil {
return nil, nil, nil, err
}
if cursor != nil && prev == nil && len(aux) > 0 {
// Cursor for previous page is calculated only when cursor is used (so, not on first page)
prev = s.collectUserCursorValues(aux[0], sortColumns...)
}
// Point cursor to the last fetched element
// if last != nil {
if fetched >= limit && limit > 0 {
next = s.collectUserCursorValues(aux[limit-1], sortColumns...)
}
if limit > 0 && uint(len(aux)) >= limit {
// we should use only as much as requested
set = append(set, aux[0:limit]...)
set = append(set, aux[:limit]...)
break
} else {
set = append(set, aux...)
@@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfUsers(
}
// @todo improve strategy for collecting next page with lower limit
// Point cursor to the last fetched element
if cursor = s.collectUserCursorValues(last, sort.Columns()...); cursor == nil {
break
}
}
if reversedCursor {
// Cursor for previous page was used
// Fetched set needs to be reverseCursor because we've forced a descending order to
// get the previous page
if reversedOrder {
// Fetched set needs to be reversed because we've forced a descending order to get the previous page
for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 {
set[i], set[j] = set[j], set[i]
}
// and flip prev/next cursors too
prev, next = next, prev
}
return set, nil
if prev != nil {
prev.Reverse = true
}
return set, prev, next, nil
}
// QueryUsers queries the database, converts and checks each row and
@@ -203,13 +215,12 @@ func (s Store) QueryUsers(
return nil, 0, nil, err
}
// If check function is set, call it and act accordingly
// check fn set, call it and see if it passed the test
// if not, skip the item
if check != nil {
if chk, err := check(res); err != nil {
return nil, 0, nil, err
} else if !chk {
// did not pass the check
// go with the next row
continue
}
}
@@ -475,7 +486,7 @@ func (Store) userColumns(aa ...string) []string {
}
}
// {true true true true true}
// {true true false true true true}
// sortableUserColumns returns all User columns flagged as sortable
//
store/tests/actionlog_test.go
+80 -85
View File
@@ -87,91 +87,86 @@ func testActionlog(t *testing.T, s store.Actionlogs) {
req.NoError(err)
req.Len(set, 5)
})
t.Run("with paging", func(t *testing.T) {
req := require.New(t)
req.NoError(s.TruncateActionlogs(ctx))
set := []*actionlog.Action{
makeNew(1, "01"),
makeNew(2, "02"),
makeNew(3, "03"),
makeNew(4, "04"),
makeNew(5, "05"),
makeNew(6, "06"),
makeNew(7, "07"),
makeNew(8, "08"),
makeNew(9, "09"),
makeNew(10, "10"),
}
req.NoError(s.CreateActionlog(ctx, set...))
f := actionlog.Filter{}
// Fetch first page
f.Limit = 3
set, f, err := store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 3)
req.NotNil(f.NextPage)
req.Nil(f.PrevPage)
req.Equal("10..8", stringifySetRange(set))
// 2nd page
f.Limit = 6
f.PageCursor = f.NextPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 6)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("7..2", stringifySetRange(set))
// 3rd, last page (1 item left)
f.Limit = 2
f.PageCursor = f.NextPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 1)
req.Nil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("1", stringifySetRange(set))
// now, in reverse, last 3 items
f.Limit = 3
f.PageCursor = f.PrevPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 3)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("4..2", stringifySetRange(set))
// still in reverse, next 6 items
f.Limit = 5
f.PageCursor = f.PrevPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 5)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("9..5", stringifySetRange(set))
// still in reverse, last 5 items (actually, we'll only get 1)
f.Limit = 5
f.PageCursor = f.PrevPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 1)
req.Nil(f.PrevPage)
req.NotNil(f.NextPage)
req.Equal("10", stringifySetRange(set))
})
})
t.Run("with keyed paging", func(t *testing.T) {
req := require.New(t)
req.NoError(s.TruncateActionlogs(ctx))
set := []*actionlog.Action{
makeNew(1, "01"),
makeNew(2, "02"),
makeNew(3, "03"),
makeNew(4, "04"),
makeNew(5, "05"),
makeNew(6, "06"),
makeNew(7, "07"),
makeNew(8, "08"),
makeNew(9, "09"),
makeNew(10, "10"),
}
req.NoError(s.CreateActionlog(ctx, set...))
f := actionlog.Filter{}
// Fetch first page
f.Limit = 3
set, f, err := store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 3)
req.NotNil(f.NextPage)
req.Nil(f.PrevPage)
req.Equal("10..8", stringifySetRange(set))
// 2nd page
f.Limit = 6
f.PageCursor = f.NextPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 6)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("7..2", stringifySetRange(set))
// 3rd, last page (1 item left)
f.Limit = 2
f.PageCursor = f.NextPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 1)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("1", stringifySetRange(set))
// try and go pass the last page
f.PageCursor = f.NextPage
set, _, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 0)
// now, in reverse, last 3 items
f.Limit = 3
f.PageCursor = f.PrevPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 3)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("3..1", stringifySetRange(set))
// still in reverse, next 6 items
f.Limit = 5
f.PageCursor = f.PrevPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 5)
req.NotNil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("9..4", stringifySetRange(set))
// still in reverse, last 5 items (actually, we'll only get 1)
f.Limit = 5
f.PageCursor = f.PrevPage
set, f, err = store.SearchActionlogs(ctx, s, f)
req.NoError(err)
req.Len(set, 1)
req.Nil(f.PrevPage)
req.NotNil(f.NextPage)
req.Equal("10", stringifySetRange(set))
})
}
store/tests/compose_records_test.go
+4 -4
View File
@@ -191,7 +191,7 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) {
t.Run("by Record", func(t *testing.T) {
req, rr := truncAndCreate(t)
rec := rr[0]
req.NoError(s.DeleteComposeRecord(ctx, mod, rec))
_, err := s.LookupComposeRecordByID(ctx, mod, rec.ID)
req.EqualError(err, store.ErrNotFound.Error())
@@ -200,7 +200,7 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) {
t.Run("by ID", func(t *testing.T) {
req, rr := truncAndCreate(t)
rec := rr[0]
req.NoError(s.DeleteComposeRecordByID(ctx, mod, rec.ID))
_, err := s.LookupComposeRecordByID(ctx, mod, rec.ID)
req.EqualError(err, store.ErrNotFound.Error())
@@ -357,15 +357,15 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) {
f.PageCursor = f.NextPage
set, f, err = s.SearchComposeRecords(ctx, mod, f)
req.Equal("v4,<NULL>;v5,<NULL>", stringifyValues(set, "str1", "str3"))
req.NoError(err)
req.Equal("v4,<NULL>;v5,<NULL>", stringifyValues(set, "str1", "str3"))
req.NoError(f.Sort.Set("str3 DESC"))
f.PageCursor = nil
f.Limit = 1
set, f, err = s.SearchComposeRecords(ctx, mod, f)
req.Equal("three,v1", stringifyValues(set, "str3", "str1"))
req.NoError(err)
req.Equal("three,v1", stringifyValues(set, "str3", "str1"))
})
})
store/tests/main_test.go
+1 -1
View File
@@ -35,7 +35,7 @@ func Test_Store(t *testing.T) {
}
)
logger.Init()
logger.SetDefault(logger.MakeDebugLogger())
var (
ctx = context.Background()
store/tests/users_test.go
+31 -9
View File
@@ -195,7 +195,7 @@ func testUsers(t *testing.T, s store.Users) {
req.Equal(prefill[0].ID, set[0].ID)
})
t.Run("with keyed paging", func(t *testing.T) {
t.Run("with paging", func(t *testing.T) {
req := require.New(t)
req.NoError(s.TruncateUsers(ctx))
@@ -214,7 +214,7 @@ func testUsers(t *testing.T, s store.Users) {
req.NoError(s.CreateUser(ctx, set...))
f := types.UserFilter{}
f.Sort = filter.SortExprSet{&filter.SortExpr{Column: "email"}}
f.Sort.Set("email")
// Fetch first page
f.Limit = 3
@@ -241,16 +241,10 @@ func testUsers(t *testing.T, s store.Users) {
set, f, err = store.SearchUsers(ctx, s, f)
req.NoError(err)
req.Len(set, 1)
req.NotNil(f.NextPage)
req.Nil(f.NextPage)
req.NotNil(f.PrevPage)
req.Equal("10", stringifySetRange(set))
// try and go pass the last page
f.PageCursor = f.NextPage
set, _, err = store.SearchUsers(ctx, s, f)
req.NoError(err)
req.Len(set, 0)
// now, in reverse, last 3 items
f.Limit = 3
f.PageCursor = f.PrevPage
@@ -310,6 +304,34 @@ func testUsers(t *testing.T, s store.Users) {
})
t.Run("with incompatible sort", func(t *testing.T) {
req := require.New(t)
req.NoError(s.TruncateUsers(ctx))
set := []*types.User{
makeNew("01"),
makeNew("02"),
makeNew("03"),
makeNew("04"),
makeNew("05"),
}
req.NoError(s.CreateUser(ctx, set...))
f := types.UserFilter{}
f.Sort = filter.SortExprSet{&filter.SortExpr{Column: "email", Descending: true}, &filter.SortExpr{Column: "handle", Descending: true}}
f.Limit = 1
set, f, err := store.SearchUsers(ctx, s, f)
req.NoError(err)
// go to next page with different sorting
f.PageCursor = f.NextPage
f.Sort = filter.SortExprSet{&filter.SortExpr{Column: "email", Descending: true}}
set, f, err = store.SearchUsers(ctx, s, f)
req.EqualError(err, "incompatible sort")
})
t.Run("by role", func(t *testing.T) {
t.Skip("not implemented")
})
system/types/attachment.go
+1 -1
View File
@@ -28,7 +28,6 @@ type (
AttachmentFilter struct {
Kind string `json:"kind,omitempty"`
Filter string `json:"filter"`
Sort string `json:"sort"`
// Check fn is called by store backend for each resource found function can
// modify the resource and return false if store should not return it
@@ -37,6 +36,7 @@ type (
Check func(*Attachment) (bool, error) `json:"-"`
// Standard helpers for paging and sorting
filter.Sorting
filter.Paging
}