diff --git a/compose/types/attachment.go b/compose/types/attachment.go index be051a21b..29a63b2f0 100644 --- a/compose/types/attachment.go +++ b/compose/types/attachment.go @@ -36,8 +36,6 @@ type ( FieldName string `json:"fieldName,omitempty"` Filter string `json:"filter"` - Sort string `json:"sort"` - // Check fn is called by store backend for each resource found function can // modify the resource and return false if store should not return it // @@ -45,6 +43,7 @@ type ( Check func(*Attachment) (bool, error) // Standard helpers for paging and sorting + filter.Sorting filter.Paging } diff --git a/pkg/codegen/assets/store_rdbms.gen.go.tpl b/pkg/codegen/assets/store_rdbms.gen.go.tpl index b4f8bef5f..20d10624e 100644 --- a/pkg/codegen/assets/store_rdbms.gen.go.tpl +++ b/pkg/codegen/assets/store_rdbms.gen.go.tpl @@ -42,6 +42,10 @@ const ( */}} {{ if $.Search.Enable }} +{{ if $.Search.Custom }} +// {{ toggleExport .Search.Export "Search" $.Types.Plural }} not generated +// {search: {custom:true}} +{{ else }} // {{ toggleExport .Search.Export "Search" $.Types.Plural }} returns all matching rows // // This function calls convert{{ export $.Types.Singular }}Filter with the given @@ -53,83 +57,78 @@ func (s Store) {{ toggleExport .Search.Export "Search" $.Types.Plural }}(ctx con q squirrel.SelectBuilder ) - -{{- if .RDBMS.CustomFilterConverter }} - q, err = s.convert{{ export $.Types.Singular }}Filter({{ template "extraArgsCallFirst" . }}f) - if err != nil { - return nil, f, err - } -{{- else }} - q = s.{{ unexport $.Types.Plural }}SelectBuilder() -{{- end }} - -{{ if $.Search.EnablePaging }} - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse -{{ end }} - -{{ if and $.Search.EnableSorting $.Search.EnablePaging }} - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } - -{{ else if $.Search.EnablePaging }} - // Sorting is disabled in definition yaml file - // {search: {enableSorting:false}} - // - // We still need to sort the results by primary key for paging purposes - curSort := filter.SortExprSet{ - {{- range $.RDBMS.Columns.PrimaryKeyFields }} - &filter.SortExpr{Column: {{ printf "%q" .Column }}, {{ if .SortDescending }}Descending: !reversedCursor, {{ end }}}, - {{- end }} - } -{{ end }} - return set, f, func() error { - {{- if $.Search.EnablePaging -}} - set, err = s.{{ unexport "fetchFullPageOf" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, curSort, f.PageCursor, f.Limit, {{ if $.Search.EnableFilterCheckFn }}f.Check{{ else }}nil{{ end }},) - + {{- if .RDBMS.CustomFilterConverter }} + q, err = s.convert{{ export $.Types.Singular }}Filter({{ template "extraArgsCallFirst" . }}f) if err != nil { return err } + {{- else }} + q = s.{{ unexport $.Types.Plural }}SelectBuilder() + {{- end }} - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collect{{ export $.Types.Singular }}CursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true + {{ if $.Search.EnablePaging }} + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil + + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err } + } - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collect{{ export $.Types.Singular }}CursorValues(set[len(set)-1], curSort.Columns()...) + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + {{- range $.RDBMS.Columns.PrimaryKeyFields }} + if f.Sort.Get({{ printf "%q" .Column }}) == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: {{ printf "%q" .Column }}, {{ if .SortDescending }}Descending: true, {{ end }}}) + } + {{- end }} + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + {{ end }} + + {{ if .RDBMS.CustomSortConverter }} + if q, err = s.{{ unexport $.Types.Plural }}Sorter({{ template "extraArgsCallFirst" . }}q, f.Sort); err != nil { + return err + } + + {{ else if $.Search.EnableSorting }} + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortable{{ export $.Types.Singular }}Columns()); err != nil { + return err + } + {{ end }} + + {{- if $.Search.EnablePaging }} + set, f.PrevPage, f.NextPage, err = s.{{ unexport "fetchFullPageOf" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, {{ if $.Search.EnableFilterCheckFn }}f.Check{{ else }}nil{{ end }},) + if err != nil { + return err } f.PageCursor = nil return nil {{- else }} - {{- if $.Search.EnableSorting }} - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, f.Sort, s.sortable{{ export $.Types.Singular }}Columns()); err != nil { - return err - } - {{ end -}} - set, _, _, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, {{ if $.Search.EnableFilterCheckFn }}f.Check{{else}}nil{{ end }}) return err - {{ end }} + {{- end }} }() } {{ end }} +{{ end }} {{ if $.Search.EnablePaging }} @@ -137,75 +136,65 @@ func (s Store) {{ toggleExport .Search.Export "Search" $.Types.Plural }}(ctx con // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural }} ( ctx context.Context{{ template "extraArgsDef" . }}, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*{{ $.Types.GoType }}) (bool, error), -) ([]*{{ $.Types.GoType }}, error) { +) (set []*{{ $.Types.GoType }}, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*{{ $.Types.GoType }}, 0, DefaultSliceCapacity) aux []*{{ $.Types.GoType }} - last *{{ $.Types.GoType }} // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) + set = make([]*{{ $.Types.GoType }}, 0, DefaultSliceCapacity) -{{ if .RDBMS.Columns.PrimaryKeyFields }} - // Make sure we always end our sort by primary keys - {{- range .RDBMS.Columns.PrimaryKeyFields }} - if sort.Get({{ printf "%q" .Column }}) == nil { - sort = append(sort, &filter.SortExpr{Column: {{ printf "%q" .Column }}}) + if cursor != nil { + cursor.Reverse = sortDesc } - {{ end }} -{{ end }} - -{{ if .RDBMS.CustomSortConverter }} - if q, err = s.{{ unexport $.Types.Plural }}Sorter({{ template "extraArgsCallFirst" . }}q, sort); err != nil { - return nil, err - } -{{ else if .Search.EnableSorting }} - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortable{{ export $.Types.Singular }}Columns()); err != nil { - return nil, err - } -{{ else if .RDBMS.Columns.PrimaryKeyFields }} - // Sort by primary keys by default - if q, err = setOrderBy(q, sort, nil); err != nil { - return nil, err - } -{{ end }} - for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collect{{ export $.Types.Singular }}CursorValues(aux[0], sortColumns...) + } + + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collect{{ export $.Types.Singular }}CursorValues(aux[limit-1], sortColumns...); } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -224,23 +213,23 @@ func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural }} ( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collect{{ export $.Types.Singular }}CursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } {{ end }} @@ -281,13 +270,12 @@ func (s Store) {{ export "query" $.Types.Plural }} ( } {{ if $.Search.EnableFilterCheckFn }} - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } diff --git a/pkg/codegen/store.go b/pkg/codegen/store.go index 7bb05c216..2356a8c65 100644 --- a/pkg/codegen/store.go +++ b/pkg/codegen/store.go @@ -175,6 +175,7 @@ type ( storeTypeSearchDef struct { Enable bool `yaml:"enable"` Export bool `yaml:"export"` + Custom bool `yaml:"custom"` EnablePaging bool `yaml:"enablePaging"` EnableSorting bool `yaml:"enableSorting"` EnableFilterCheckFn bool `yaml:"enableFilterCheckFunction"` @@ -225,6 +226,7 @@ func procStore(mm ...string) ([]*storeDef, error) { Search: storeTypeSearchDef{ Enable: true, Export: true, + Custom: false, EnablePaging: true, EnableSorting: true, EnableFilterCheckFn: true, diff --git a/pkg/filter/pagination.go b/pkg/filter/pagination.go index 84aa5d182..1881cd885 100644 --- a/pkg/filter/pagination.go +++ b/pkg/filter/pagination.go @@ -77,7 +77,11 @@ func (p *PagingCursor) String() string { var o = "<" for i, key := range p.keys { - o += fmt.Sprintf("%s: %v, ", key, p.values[i]) + o += fmt.Sprintf("%s: %v", key, p.values[i]) + if p.desc[i] { + o += " DESC" + } + o += ", " } if p.Reverse { @@ -160,6 +164,42 @@ func (p *PagingCursor) Decode(cursor string) error { return err } +// Sort returns: +// - sort if cursor is nil +// - sort from cursor when sort is empty +// - sort from cursor when sort is compatible with cursor +// - error if sort & cursor are incompatible +func (p *PagingCursor) Sort(sort SortExprSet) (SortExprSet, error) { + if p == nil { + return sort, nil + } + + if len(sort) == 0 { + // sort emprt, create it from cursor + for k := range p.keys { + sort = append(sort, &SortExpr{ + Column: p.keys[k], + Descending: p.desc[k], + }) + } + return sort, nil + } + + // check compatibility + ss := sort.Columns() + if len(p.keys) != len(ss) { + return nil, fmt.Errorf("incompatible sort") + } + + for k := range p.keys { + if p.keys[k] != ss[k] { + return nil, fmt.Errorf("incompatible sort") + } + } + + return sort, nil +} + func parseCursor(in string) (p *PagingCursor, err error) { if len(in) == 0 { return nil, nil diff --git a/pkg/filter/pagination_test.go b/pkg/filter/pagination_test.go index 523e5d788..d5c0da2d5 100644 --- a/pkg/filter/pagination_test.go +++ b/pkg/filter/pagination_test.go @@ -19,10 +19,10 @@ func Test_cursorEncDec(t *testing.T) { { cur.Set("uint64", id, true) - cur.Set("string", "foo", true) + cur.Set("string", "foo", false) req.Len(cur.values, 2) req.Equal(id, cur.values[0]) - req.Equal(fmt.Sprintf("", id), cur.String()) + req.Equal(fmt.Sprintf("", id), cur.String()) } { @@ -34,7 +34,7 @@ func Test_cursorEncDec(t *testing.T) { req.Len(dec.values, 2) req.Equal(id, dec.values[0]) req.Equal("foo", dec.values[1]) - req.Equal(fmt.Sprintf("", id), cur.String()) + req.Equal(fmt.Sprintf("", id), cur.String()) } } diff --git a/pkg/filter/sorting.go b/pkg/filter/sorting.go index 40edd8cf3..4a408b2c6 100644 --- a/pkg/filter/sorting.go +++ b/pkg/filter/sorting.go @@ -125,6 +125,15 @@ func (set SortExprSet) Reverse() { } } +// Sorting is revered if 1st expr has desc direction +func (set SortExprSet) Reversed() bool { + if len(set) > 0 { + return set[0].Descending + } + + return false +} + // Reverse reverses direction on each expression func (set SortExprSet) Columns() []string { out := make([]string, len(set)) diff --git a/store/actionlog.yaml b/store/actionlog.yaml index 9ef468f4e..9a26f29c2 100644 --- a/store/actionlog.yaml +++ b/store/actionlog.yaml @@ -8,7 +8,7 @@ types: filterType: actionlog.Filter fields: - - { field: ID, sortDescending: true } + - { field: ID, sortable: true, sortDescending: true } - { field: Timestamp, type: "time.Time" } - { field: RequestOrigin } - { field: RequestID } @@ -33,7 +33,6 @@ rdbms: ActorID: { column: actor_id } search: - enableSorting: false enableFilterCheckFunction: false update: diff --git a/store/compose_records.yaml b/store/compose_records.yaml index 18798938c..98299ee0e 100644 --- a/store/compose_records.yaml +++ b/store/compose_records.yaml @@ -53,6 +53,7 @@ rdbms: ModuleID: { column: module_id } search: + custom: true export: false create: diff --git a/store/rdbms/actionlog.gen.go b/store/rdbms/actionlog.gen.go index 195a00fdf..2b2efaa41 100644 --- a/store/rdbms/actionlog.gen.go +++ b/store/rdbms/actionlog.gen.go @@ -30,42 +30,50 @@ func (s Store) SearchActionlogs(ctx context.Context, f actionlog.Filter) (action set []*actionlog.Action q squirrel.SelectBuilder ) - q, err = s.convertActionlogFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting is disabled in definition yaml file - // {search: {enableSorting:false}} - // - // We still need to sort the results by primary key for paging purposes - curSort := filter.SortExprSet{ - &filter.SortExpr{Column: "id", Descending: !reversedCursor}, - } return set, f, func() error { - set, err = s.fetchFullPageOfActionlogs(ctx, q, curSort, f.PageCursor, f.Limit, nil) - + q, err = s.convertActionlogFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectActionlogCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectActionlogCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id", Descending: true}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableActionlogColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfActionlogs(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, nil) + if err != nil { + return err } f.PageCursor = nil @@ -77,58 +85,64 @@ func (s Store) SearchActionlogs(ctx context.Context, f actionlog.Filter) (action // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfActionlogs( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*actionlog.Action) (bool, error), -) ([]*actionlog.Action, error) { +) (set []*actionlog.Action, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*actionlog.Action, 0, DefaultSliceCapacity) - aux []*actionlog.Action - last *actionlog.Action + aux []*actionlog.Action // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*actionlog.Action, 0, DefaultSliceCapacity) - // Sort by primary keys by default - if q, err = setOrderBy(q, sort, nil); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryActionlogs(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryActionlogs(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectActionlogCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectActionlogCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -147,23 +161,23 @@ func (s Store) fetchFullPageOfActionlogs( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectActionlogCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryActionlogs queries the database, converts and checks each row and @@ -315,7 +329,16 @@ func (Store) actionlogColumns(aa ...string) []string { } } -// {true true true false false} +// {true true false true true false} + +// sortableActionlogColumns returns all Actionlog columns flagged as sortable +// +// With optional string arg, all columns are returned aliased +func (Store) sortableActionlogColumns() map[string]string { + return map[string]string{ + "id": "id", + } +} // internalActionlogEncoder encodes fields from actionlog.Action to store.Payload (map) // diff --git a/store/rdbms/applications.gen.go b/store/rdbms/applications.gen.go index 141a28e02..609f467fb 100644 --- a/store/rdbms/applications.gen.go +++ b/store/rdbms/applications.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchApplications(ctx context.Context, f types.ApplicationFilter set []*types.Application q squirrel.SelectBuilder ) - q, err = s.convertApplicationFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfApplications(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertApplicationFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectApplicationCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectApplicationCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableApplicationColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfApplications(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchApplications(ctx context.Context, f types.ApplicationFilter // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfApplications( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Application) (bool, error), -) ([]*types.Application, error) { +) (set []*types.Application, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Application, 0, DefaultSliceCapacity) - aux []*types.Application - last *types.Application + aux []*types.Application // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Application, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableApplicationColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryApplications(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryApplications(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectApplicationCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectApplicationCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfApplications( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectApplicationCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryApplications queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryApplications( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -434,7 +445,7 @@ func (Store) applicationColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableApplicationColumns returns all Application columns flagged as sortable // diff --git a/store/rdbms/attachments.gen.go b/store/rdbms/attachments.gen.go index 1b443d907..e52999bdd 100644 --- a/store/rdbms/attachments.gen.go +++ b/store/rdbms/attachments.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchAttachments(ctx context.Context, f types.AttachmentFilter) set []*types.Attachment q squirrel.SelectBuilder ) - q, err = s.convertAttachmentFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertAttachmentFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryAttachments(ctx, q, f.Check) return err - }() } @@ -76,13 +76,12 @@ func (s Store) QueryAttachments( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -311,7 +310,7 @@ func (Store) attachmentColumns(aa ...string) []string { } } -// {true true false false true} +// {true true false false false true} // internalAttachmentEncoder encodes fields from types.Attachment to store.Payload (map) // diff --git a/store/rdbms/compose_attachments.gen.go b/store/rdbms/compose_attachments.gen.go index 51b07d315..8ea76f8cd 100644 --- a/store/rdbms/compose_attachments.gen.go +++ b/store/rdbms/compose_attachments.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchComposeAttachments(ctx context.Context, f types.AttachmentF set []*types.Attachment q squirrel.SelectBuilder ) - q, err = s.convertComposeAttachmentFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertComposeAttachmentFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryComposeAttachments(ctx, q, f.Check) return err - }() } @@ -76,13 +76,12 @@ func (s Store) QueryComposeAttachments( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -313,7 +312,7 @@ func (Store) composeAttachmentColumns(aa ...string) []string { } } -// {true true false false true} +// {true true false false false true} // internalComposeAttachmentEncoder encodes fields from types.Attachment to store.Payload (map) // diff --git a/store/rdbms/compose_charts.gen.go b/store/rdbms/compose_charts.gen.go index 78639550d..e9b8c9f88 100644 --- a/store/rdbms/compose_charts.gen.go +++ b/store/rdbms/compose_charts.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchComposeCharts(ctx context.Context, f types.ChartFilter) (ty set []*types.Chart q squirrel.SelectBuilder ) - q, err = s.convertComposeChartFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfComposeCharts(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertComposeChartFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectComposeChartCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeChartCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableComposeChartColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeCharts(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchComposeCharts(ctx context.Context, f types.ChartFilter) (ty // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfComposeCharts( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Chart) (bool, error), -) ([]*types.Chart, error) { +) (set []*types.Chart, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Chart, 0, DefaultSliceCapacity) - aux []*types.Chart - last *types.Chart + aux []*types.Chart // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Chart, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableComposeChartColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryComposeCharts(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryComposeCharts(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectComposeChartCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectComposeChartCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposeCharts( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectComposeChartCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryComposeCharts queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryComposeCharts( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -444,7 +455,7 @@ func (Store) composeChartColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableComposeChartColumns returns all ComposeChart columns flagged as sortable // diff --git a/store/rdbms/compose_module_fields.gen.go b/store/rdbms/compose_module_fields.gen.go index 5637b8003..275939104 100644 --- a/store/rdbms/compose_module_fields.gen.go +++ b/store/rdbms/compose_module_fields.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchComposeModuleFields(ctx context.Context, f types.ModuleFiel set []*types.ModuleField q squirrel.SelectBuilder ) - q, err = s.convertComposeModuleFieldFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertComposeModuleFieldFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryComposeModuleFields(ctx, q, nil) return err - }() } @@ -311,7 +311,7 @@ func (Store) composeModuleFieldColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalComposeModuleFieldEncoder encodes fields from types.ModuleField to store.Payload (map) // diff --git a/store/rdbms/compose_modules.gen.go b/store/rdbms/compose_modules.gen.go index b79df414a..d22730386 100644 --- a/store/rdbms/compose_modules.gen.go +++ b/store/rdbms/compose_modules.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchComposeModules(ctx context.Context, f types.ModuleFilter) ( set []*types.Module q squirrel.SelectBuilder ) - q, err = s.convertComposeModuleFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfComposeModules(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertComposeModuleFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectComposeModuleCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeModuleCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableComposeModuleColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeModules(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchComposeModules(ctx context.Context, f types.ModuleFilter) ( // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfComposeModules( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Module) (bool, error), -) ([]*types.Module, error) { +) (set []*types.Module, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Module, 0, DefaultSliceCapacity) - aux []*types.Module - last *types.Module + aux []*types.Module // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Module, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableComposeModuleColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryComposeModules(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryComposeModules(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectComposeModuleCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectComposeModuleCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposeModules( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectComposeModuleCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryComposeModules queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryComposeModules( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -454,7 +465,7 @@ func (Store) composeModuleColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableComposeModuleColumns returns all ComposeModule columns flagged as sortable // diff --git a/store/rdbms/compose_namespaces.gen.go b/store/rdbms/compose_namespaces.gen.go index a8ad2e35b..1ed0a03cb 100644 --- a/store/rdbms/compose_namespaces.gen.go +++ b/store/rdbms/compose_namespaces.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchComposeNamespaces(ctx context.Context, f types.NamespaceFil set []*types.Namespace q squirrel.SelectBuilder ) - q, err = s.convertComposeNamespaceFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfComposeNamespaces(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertComposeNamespaceFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectComposeNamespaceCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeNamespaceCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableComposeNamespaceColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeNamespaces(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchComposeNamespaces(ctx context.Context, f types.NamespaceFil // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfComposeNamespaces( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Namespace) (bool, error), -) ([]*types.Namespace, error) { +) (set []*types.Namespace, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Namespace, 0, DefaultSliceCapacity) - aux []*types.Namespace - last *types.Namespace + aux []*types.Namespace // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Namespace, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableComposeNamespaceColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryComposeNamespaces(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryComposeNamespaces(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectComposeNamespaceCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectComposeNamespaceCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposeNamespaces( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectComposeNamespaceCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryComposeNamespaces queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryComposeNamespaces( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -443,7 +454,7 @@ func (Store) composeNamespaceColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableComposeNamespaceColumns returns all ComposeNamespace columns flagged as sortable // diff --git a/store/rdbms/compose_pages.gen.go b/store/rdbms/compose_pages.gen.go index 2f19257f8..7b01d80ae 100644 --- a/store/rdbms/compose_pages.gen.go +++ b/store/rdbms/compose_pages.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchComposePages(ctx context.Context, f types.PageFilter) (type set []*types.Page q squirrel.SelectBuilder ) - q, err = s.convertComposePageFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfComposePages(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertComposePageFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectComposePageCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposePageCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableComposePageColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposePages(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchComposePages(ctx context.Context, f types.PageFilter) (type // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfComposePages( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Page) (bool, error), -) ([]*types.Page, error) { +) (set []*types.Page, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Page, 0, DefaultSliceCapacity) - aux []*types.Page - last *types.Page + aux []*types.Page // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Page, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableComposePageColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryComposePages(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryComposePages(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectComposePageCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectComposePageCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfComposePages( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectComposePageCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryComposePages queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryComposePages( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -464,7 +475,7 @@ func (Store) composePageColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableComposePageColumns returns all ComposePage columns flagged as sortable // diff --git a/store/rdbms/compose_record_values.gen.go b/store/rdbms/compose_record_values.gen.go index 3bd60a439..7e2c528d3 100644 --- a/store/rdbms/compose_record_values.gen.go +++ b/store/rdbms/compose_record_values.gen.go @@ -29,15 +29,15 @@ func (s Store) searchComposeRecordValues(ctx context.Context, _mod *types.Module set []*types.RecordValue q squirrel.SelectBuilder ) - q, err = s.convertComposeRecordValueFilter(_mod, f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertComposeRecordValueFilter(_mod, f) + if err != nil { + return err + } + set, _, _, err = s.QueryComposeRecordValues(ctx, _mod, q, nil) return err - }() } @@ -287,7 +287,7 @@ func (Store) composeRecordValueColumns(aa ...string) []string { } } -// {true false false false false} +// {true false false false false false} // internalComposeRecordValueEncoder encodes fields from types.RecordValue to store.Payload (map) // diff --git a/store/rdbms/compose_records.gen.go b/store/rdbms/compose_records.gen.go index 3a97ef0dc..860894413 100644 --- a/store/rdbms/compose_records.gen.go +++ b/store/rdbms/compose_records.gen.go @@ -20,116 +20,71 @@ import ( var _ = errors.Is -// searchComposeRecords returns all matching rows -// -// This function calls convertComposeRecordFilter with the given -// types.RecordFilter and expects to receive a working squirrel.SelectBuilder -func (s Store) searchComposeRecords(ctx context.Context, _mod *types.Module, f types.RecordFilter) (types.RecordSet, types.RecordFilter, error) { - var ( - err error - set []*types.Record - q squirrel.SelectBuilder - ) - q, err = s.convertComposeRecordFilter(_mod, f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } - - return set, f, func() error { - set, err = s.fetchFullPageOfComposeRecords(ctx, _mod, q, curSort, f.PageCursor, f.Limit, f.Check) - - if err != nil { - return err - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectComposeRecordCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeRecordCursorValues(set[len(set)-1], curSort.Columns()...) - } - - f.PageCursor = nil - return nil - }() -} +// searchComposeRecords not generated +// {search: {custom:true}} // fetchFullPageOfComposeRecords collects all requested results. // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfComposeRecords( ctx context.Context, _mod *types.Module, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Record) (bool, error), -) ([]*types.Record, error) { +) (set []*types.Record, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Record, 0, DefaultSliceCapacity) - aux []*types.Record - last *types.Record + aux []*types.Record // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Record, 0, DefaultSliceCapacity) - if q, err = s.composeRecordsSorter(_mod, q, sort); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryComposeRecords(ctx, _mod, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryComposeRecords(ctx, _mod, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectComposeRecordCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectComposeRecordCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -148,23 +103,23 @@ func (s Store) fetchFullPageOfComposeRecords( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectComposeRecordCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryComposeRecords queries the database, converts and checks each row and @@ -202,13 +157,12 @@ func (s Store) QueryComposeRecords( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -444,7 +398,7 @@ func (Store) composeRecordColumns(aa ...string) []string { } } -// {true false true true true} +// {true false true true true true} // sortableComposeRecordColumns returns all ComposeRecord columns flagged as sortable // diff --git a/store/rdbms/compose_records.go b/store/rdbms/compose_records.go index 395b91be1..f405537c1 100644 --- a/store/rdbms/compose_records.go +++ b/store/rdbms/compose_records.go @@ -22,14 +22,40 @@ const ( // physical columns are part of module-field configuration // SearchComposeRecords returns all matching ComposeRecords from store -func (s Store) SearchComposeRecords(ctx context.Context, m *types.Module, filter types.RecordFilter) (set types.RecordSet, f types.RecordFilter, err error) { - // In when module requires this, - set, f, err = s.searchComposeRecords(ctx, m, filter) - if err != nil { - return - } +func (s Store) SearchComposeRecords(ctx context.Context, m *types.Module, f types.RecordFilter) (types.RecordSet, types.RecordFilter, error) { + var ( + err error + set []*types.Record + q squirrel.SelectBuilder + ) - return + return set, f, func() error { + q, err = s.convertComposeRecordFilter(m, f) + if err != nil { + return err + } + + f.PrevPage, f.NextPage = nil, nil + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + if q, err = s.composeRecordsSorter(m, q, f.Sort); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfComposeRecords(ctx, m, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err + } + + f.PageCursor = nil + return nil + }() } // LookupComposeRecordByID searches for compose record by ID @@ -351,7 +377,7 @@ func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string) collect = func(cc ...string) { for _, c := range cc { switch c { - case "crd.id": + case "id": cursor.Set(c, res.ID, false) pkID = true case "crd.created_at": @@ -374,7 +400,7 @@ func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string) collect(cc...) if !hasUnique || !pkID { collect( - "crd.id", + "id", ) } diff --git a/store/rdbms/credentials.gen.go b/store/rdbms/credentials.gen.go index 9e1da3071..ab0896cb9 100644 --- a/store/rdbms/credentials.gen.go +++ b/store/rdbms/credentials.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchCredentials(ctx context.Context, f types.CredentialsFilter) set []*types.Credentials q squirrel.SelectBuilder ) - q, err = s.convertCredentialsFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertCredentialsFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryCredentials(ctx, q, nil) return err - }() } @@ -302,7 +302,7 @@ func (Store) credentialsColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalCredentialsEncoder encodes fields from types.Credentials to store.Payload (map) // diff --git a/store/rdbms/labels.gen.go b/store/rdbms/labels.gen.go index 4774b0630..04814751f 100644 --- a/store/rdbms/labels.gen.go +++ b/store/rdbms/labels.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchLabels(ctx context.Context, f types.LabelFilter) (types.Lab set []*types.Label q squirrel.SelectBuilder ) - q, err = s.convertLabelFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertLabelFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryLabels(ctx, q, nil) return err - }() } @@ -292,7 +292,7 @@ func (Store) labelColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalLabelEncoder encodes fields from types.Label to store.Payload (map) // diff --git a/store/rdbms/messaging_attachments.gen.go b/store/rdbms/messaging_attachments.gen.go index 8a8a9d2fb..5e5e38f02 100644 --- a/store/rdbms/messaging_attachments.gen.go +++ b/store/rdbms/messaging_attachments.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchMessagingAttachments(ctx context.Context, f types.Attachmen set []*types.Attachment q squirrel.SelectBuilder ) - q, err = s.convertMessagingAttachmentFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertMessagingAttachmentFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryMessagingAttachments(ctx, q, f.Check) return err - }() } @@ -76,13 +76,12 @@ func (s Store) QueryMessagingAttachments( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -309,7 +308,7 @@ func (Store) messagingAttachmentColumns(aa ...string) []string { } } -// {true true false false true} +// {true true false false false true} // internalMessagingAttachmentEncoder encodes fields from types.Attachment to store.Payload (map) // diff --git a/store/rdbms/messaging_channel_members.gen.go b/store/rdbms/messaging_channel_members.gen.go index 6b9b55e90..377db127c 100644 --- a/store/rdbms/messaging_channel_members.gen.go +++ b/store/rdbms/messaging_channel_members.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchMessagingChannelMembers(ctx context.Context, f types.Channe set []*types.ChannelMember q squirrel.SelectBuilder ) - q, err = s.convertMessagingChannelMemberFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertMessagingChannelMemberFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryMessagingChannelMembers(ctx, q, nil) return err - }() } @@ -285,7 +285,7 @@ func (Store) messagingChannelMemberColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalMessagingChannelMemberEncoder encodes fields from types.ChannelMember to store.Payload (map) // diff --git a/store/rdbms/messaging_channels.gen.go b/store/rdbms/messaging_channels.gen.go index ed2fd7814..f4993463c 100644 --- a/store/rdbms/messaging_channels.gen.go +++ b/store/rdbms/messaging_channels.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchMessagingChannels(ctx context.Context, f types.ChannelFilte set []*types.Channel q squirrel.SelectBuilder ) - q, err = s.convertMessagingChannelFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfMessagingChannels(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertMessagingChannelFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectMessagingChannelCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectMessagingChannelCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableMessagingChannelColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfMessagingChannels(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchMessagingChannels(ctx context.Context, f types.ChannelFilte // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfMessagingChannels( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Channel) (bool, error), -) ([]*types.Channel, error) { +) (set []*types.Channel, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Channel, 0, DefaultSliceCapacity) - aux []*types.Channel - last *types.Channel + aux []*types.Channel // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Channel, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableMessagingChannelColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryMessagingChannels(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryMessagingChannels(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectMessagingChannelCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectMessagingChannelCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfMessagingChannels( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectMessagingChannelCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryMessagingChannels queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryMessagingChannels( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -442,7 +453,7 @@ func (Store) messagingChannelColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableMessagingChannelColumns returns all MessagingChannel columns flagged as sortable // diff --git a/store/rdbms/messaging_flags.gen.go b/store/rdbms/messaging_flags.gen.go index e84978930..5d051c319 100644 --- a/store/rdbms/messaging_flags.gen.go +++ b/store/rdbms/messaging_flags.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchMessagingFlags(ctx context.Context, f types.MessageFlagFilt set []*types.MessageFlag q squirrel.SelectBuilder ) - q, err = s.convertMessagingFlagFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertMessagingFlagFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryMessagingFlags(ctx, q, nil) return err - }() } @@ -290,7 +290,7 @@ func (Store) messagingFlagColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalMessagingFlagEncoder encodes fields from types.MessageFlag to store.Payload (map) // diff --git a/store/rdbms/messaging_mentions.gen.go b/store/rdbms/messaging_mentions.gen.go index 510d727cf..2ae6b8ebe 100644 --- a/store/rdbms/messaging_mentions.gen.go +++ b/store/rdbms/messaging_mentions.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchMessagingMentions(ctx context.Context, f types.MentionFilte set []*types.Mention q squirrel.SelectBuilder ) - q, err = s.convertMessagingMentionFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertMessagingMentionFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryMessagingMentions(ctx, q, nil) return err - }() } @@ -292,7 +292,7 @@ func (Store) messagingMentionColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalMessagingMentionEncoder encodes fields from types.Mention to store.Payload (map) // diff --git a/store/rdbms/messaging_message_attachments.gen.go b/store/rdbms/messaging_message_attachments.gen.go index 3a9fdf377..537235425 100644 --- a/store/rdbms/messaging_message_attachments.gen.go +++ b/store/rdbms/messaging_message_attachments.gen.go @@ -260,7 +260,7 @@ func (Store) messagingMessageAttachmentColumns(aa ...string) []string { } } -// {false true false false false} +// {false true false false false false} // internalMessagingMessageAttachmentEncoder encodes fields from types.MessageAttachment to store.Payload (map) // diff --git a/store/rdbms/messaging_messages.gen.go b/store/rdbms/messaging_messages.gen.go index 36ebdbf47..6455c5b1f 100644 --- a/store/rdbms/messaging_messages.gen.go +++ b/store/rdbms/messaging_messages.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchMessagingMessages(ctx context.Context, f types.MessageFilte set []*types.Message q squirrel.SelectBuilder ) - q, err = s.convertMessagingMessageFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertMessagingMessageFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QueryMessagingMessages(ctx, q, nil) return err - }() } @@ -302,7 +302,7 @@ func (Store) messagingMessageColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalMessagingMessageEncoder encodes fields from types.Message to store.Payload (map) // diff --git a/store/rdbms/messaging_unread.gen.go b/store/rdbms/messaging_unread.gen.go index 53f18ba09..dc313a298 100644 --- a/store/rdbms/messaging_unread.gen.go +++ b/store/rdbms/messaging_unread.gen.go @@ -263,7 +263,7 @@ func (Store) messagingUnreadColumns(aa ...string) []string { } } -// {false true false false false} +// {false true false false false false} // internalMessagingUnreadEncoder encodes fields from types.Unread to store.Payload (map) // diff --git a/store/rdbms/rbac_rules.gen.go b/store/rdbms/rbac_rules.gen.go index b95ecc372..dd770dbbd 100644 --- a/store/rdbms/rbac_rules.gen.go +++ b/store/rdbms/rbac_rules.gen.go @@ -29,12 +29,12 @@ func (s Store) SearchRbacRules(ctx context.Context, f rbac.RuleFilter) (rbac.Rul set []*rbac.Rule q squirrel.SelectBuilder ) - q = s.rbacRulesSelectBuilder() return set, f, func() error { + q = s.rbacRulesSelectBuilder() + set, _, _, err = s.QueryRbacRules(ctx, q, nil) return err - }() } @@ -280,7 +280,7 @@ func (Store) rbacRuleColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalRbacRuleEncoder encodes fields from rbac.Rule to store.Payload (map) // diff --git a/store/rdbms/rdbms.go b/store/rdbms/rdbms.go index 51b7ad5ce..cc885e37f 100644 --- a/store/rdbms/rdbms.go +++ b/store/rdbms/rdbms.go @@ -409,8 +409,11 @@ func tx(ctx context.Context, dbCandidate interface{}, cfg *Config, txOpt *sql.Tx func setCursorCond(q squirrel.SelectBuilder, cursor *filter.PagingCursor) squirrel.SelectBuilder { if cursor != nil && len(cursor.Keys()) > 0 { const cursorTpl = `(%s) %s (?%s)` + + //expecting row values to be greater then args op := ">" if cursor.Reverse { + //expecting row values to be lesser then args op = "<" } diff --git a/store/rdbms/reminders.gen.go b/store/rdbms/reminders.gen.go index fb8462743..9dc001068 100644 --- a/store/rdbms/reminders.gen.go +++ b/store/rdbms/reminders.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchReminders(ctx context.Context, f types.ReminderFilter) (typ set []*types.Reminder q squirrel.SelectBuilder ) - q, err = s.convertReminderFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfReminders(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertReminderFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectReminderCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectReminderCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableReminderColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfReminders(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchReminders(ctx context.Context, f types.ReminderFilter) (typ // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfReminders( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Reminder) (bool, error), -) ([]*types.Reminder, error) { +) (set []*types.Reminder, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Reminder, 0, DefaultSliceCapacity) - aux []*types.Reminder - last *types.Reminder + aux []*types.Reminder // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Reminder, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableReminderColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryReminders(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryReminders(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectReminderCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectReminderCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfReminders( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectReminderCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryReminders queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryReminders( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -444,7 +455,7 @@ func (Store) reminderColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableReminderColumns returns all Reminder columns flagged as sortable // diff --git a/store/rdbms/role_members.gen.go b/store/rdbms/role_members.gen.go index 616b1baaa..fc4bf9689 100644 --- a/store/rdbms/role_members.gen.go +++ b/store/rdbms/role_members.gen.go @@ -29,12 +29,12 @@ func (s Store) SearchRoleMembers(ctx context.Context, f types.RoleMemberFilter) set []*types.RoleMember q squirrel.SelectBuilder ) - q = s.roleMembersSelectBuilder() return set, f, func() error { + q = s.roleMembersSelectBuilder() + set, _, _, err = s.QueryRoleMembers(ctx, q, nil) return err - }() } @@ -274,7 +274,7 @@ func (Store) roleMemberColumns(aa ...string) []string { } } -// {true true false false false} +// {true true false false false false} // internalRoleMemberEncoder encodes fields from types.RoleMember to store.Payload (map) // diff --git a/store/rdbms/roles.gen.go b/store/rdbms/roles.gen.go index 70af894b4..35567ab08 100644 --- a/store/rdbms/roles.gen.go +++ b/store/rdbms/roles.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchRoles(ctx context.Context, f types.RoleFilter) (types.RoleS set []*types.Role q squirrel.SelectBuilder ) - q, err = s.convertRoleFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfRoles(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertRoleFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectRoleCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectRoleCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableRoleColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfRoles(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchRoles(ctx context.Context, f types.RoleFilter) (types.RoleS // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfRoles( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.Role) (bool, error), -) ([]*types.Role, error) { +) (set []*types.Role, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.Role, 0, DefaultSliceCapacity) - aux []*types.Role - last *types.Role + aux []*types.Role // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.Role, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableRoleColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryRoles(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryRoles(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectRoleCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectRoleCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfRoles( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectRoleCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryRoles queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryRoles( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -456,7 +467,7 @@ func (Store) roleColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableRoleColumns returns all Role columns flagged as sortable // diff --git a/store/rdbms/settings.gen.go b/store/rdbms/settings.gen.go index 861c5d310..13205818b 100644 --- a/store/rdbms/settings.gen.go +++ b/store/rdbms/settings.gen.go @@ -29,15 +29,15 @@ func (s Store) SearchSettings(ctx context.Context, f types.SettingsFilter) (type set []*types.SettingValue q squirrel.SelectBuilder ) - q, err = s.convertSettingFilter(f) - if err != nil { - return nil, f, err - } return set, f, func() error { + q, err = s.convertSettingFilter(f) + if err != nil { + return err + } + set, _, _, err = s.QuerySettings(ctx, q, f.Check) return err - }() } @@ -76,13 +76,12 @@ func (s Store) QuerySettings( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -302,7 +301,7 @@ func (Store) settingColumns(aa ...string) []string { } } -// {true true false false true} +// {true true false false false true} // internalSettingEncoder encodes fields from types.SettingValue to store.Payload (map) // diff --git a/store/rdbms/users.gen.go b/store/rdbms/users.gen.go index 3cc6fc85c..14425f82c 100644 --- a/store/rdbms/users.gen.go +++ b/store/rdbms/users.gen.go @@ -30,44 +30,50 @@ func (s Store) SearchUsers(ctx context.Context, f types.UserFilter) (types.UserS set []*types.User q squirrel.SelectBuilder ) - q, err = s.convertUserFilter(f) - if err != nil { - return nil, f, err - } - - // Cleanup anything we've accidentally received... - f.PrevPage, f.NextPage = nil, nil - - // When cursor for a previous page is used it's marked as reversed - // This tells us to flip the descending flag on all used sort keys - reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse - - // Sorting and paging are both enabled in definition yaml file - // {search: {enableSorting:true, enablePaging:true}} - curSort := f.Sort.Clone() - - // If paging with reverse cursor, change the sorting - // direction for all columns we're sorting by - if reversedCursor { - curSort.Reverse() - } return set, f, func() error { - set, err = s.fetchFullPageOfUsers(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) - + q, err = s.convertUserFilter(f) if err != nil { return err } - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { - f.PrevPage = s.collectUserCursorValues(set[0], curSort.Columns()...) - f.PrevPage.Reverse = true - } + // Paging enabled + // {search: {enablePaging:true}} + // Cleanup unwanted cursors (only relevant is f.PageCursor, next&prev are reset and returned) + f.PrevPage, f.NextPage = nil, nil - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectUserCursorValues(set[len(set)-1], curSort.Columns()...) + if f.PageCursor != nil { + // Page cursor exists so we need to validate it against used sort + if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil { + return err + } + } + + if len(f.Sort) == 0 { + f.Sort = filter.SortExprSet{} + } + + // Make sure results are always sorted at least by primary keys + if f.Sort.Get("id") == nil { + f.Sort = append(f.Sort, &filter.SortExpr{Column: "id"}) + } + + sort := f.Sort.Clone() + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + if f.PageCursor != nil && f.PageCursor.Reverse { + sort.Reverse() + } + + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortableUserColumns()); err != nil { + return err + } + + set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfUsers(ctx, q, sort.Columns(), sort.Reversed(), f.PageCursor, f.Limit, f.Check) + if err != nil { + return err } f.PageCursor = nil @@ -79,58 +85,64 @@ func (s Store) SearchUsers(ctx context.Context, f types.UserFilter) (types.UserS // // Function applies: // - cursor conditions (where ...) -// - sorting rules (order by ...) // - limit // // Main responsibility of this function is to perform additional sequential queries in case when not enough results -// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +// are collected due to failed check on a specific row (by check fn). +// +// Function then moves cursor to the last item fetched func (s Store) fetchFullPageOfUsers( ctx context.Context, q squirrel.SelectBuilder, - sort filter.SortExprSet, + sortColumns []string, + sortDesc bool, cursor *filter.PagingCursor, limit uint, check func(*types.User) (bool, error), -) ([]*types.User, error) { +) (set []*types.User, prev, next *filter.PagingCursor, err error) { var ( - set = make([]*types.User, 0, DefaultSliceCapacity) - aux []*types.User - last *types.User + aux []*types.User // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reversedCursor = cursor != nil && cursor.Reverse + reversedOrder = cursor != nil && cursor.Reverse // copy of the select builder tryQuery squirrel.SelectBuilder fetched uint - err error ) - // Make sure we always end our sort by primary keys - if sort.Get("id") == nil { - sort = append(sort, &filter.SortExpr{Column: "id"}) - } + set = make([]*types.User, 0, DefaultSliceCapacity) - // Apply sorting expr from filter to query - if q, err = setOrderBy(q, sort, s.sortableUserColumns()); err != nil { - return nil, err + if cursor != nil { + cursor.Reverse = sortDesc } for try := 0; try < MaxRefetches; try++ { tryQuery = setCursorCond(q, cursor) if limit > 0 { - tryQuery = tryQuery.Limit(uint64(limit)) + tryQuery = tryQuery.Limit(uint64(limit + 1)) } - if aux, fetched, last, err = s.QueryUsers(ctx, tryQuery, check); err != nil { - return nil, err + if aux, fetched, _, err = s.QueryUsers(ctx, tryQuery, check); err != nil { + return nil, nil, nil, err + } + + if cursor != nil && prev == nil && len(aux) > 0 { + // Cursor for previous page is calculated only when cursor is used (so, not on first page) + prev = s.collectUserCursorValues(aux[0], sortColumns...) + } + + // Point cursor to the last fetched element + // if last != nil { + if fetched >= limit && limit > 0 { + next = s.collectUserCursorValues(aux[limit-1], sortColumns...) } if limit > 0 && uint(len(aux)) >= limit { // we should use only as much as requested - set = append(set, aux[0:limit]...) + set = append(set, aux[:limit]...) break } else { set = append(set, aux...) @@ -149,23 +161,23 @@ func (s Store) fetchFullPageOfUsers( } // @todo improve strategy for collecting next page with lower limit - - // Point cursor to the last fetched element - if cursor = s.collectUserCursorValues(last, sort.Columns()...); cursor == nil { - break - } } - if reversedCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previous page + if reversedOrder { + // Fetched set needs to be reversed because we've forced a descending order to get the previous page for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { set[i], set[j] = set[j], set[i] } + + // and flip prev/next cursors too + prev, next = next, prev } - return set, nil + if prev != nil { + prev.Reverse = true + } + + return set, prev, next, nil } // QueryUsers queries the database, converts and checks each row and @@ -203,13 +215,12 @@ func (s Store) QueryUsers( return nil, 0, nil, err } - // If check function is set, call it and act accordingly + // check fn set, call it and see if it passed the test + // if not, skip the item if check != nil { if chk, err := check(res); err != nil { return nil, 0, nil, err } else if !chk { - // did not pass the check - // go with the next row continue } } @@ -475,7 +486,7 @@ func (Store) userColumns(aa ...string) []string { } } -// {true true true true true} +// {true true false true true true} // sortableUserColumns returns all User columns flagged as sortable // diff --git a/store/tests/actionlog_test.go b/store/tests/actionlog_test.go index 873ed7a6a..96d832f2e 100644 --- a/store/tests/actionlog_test.go +++ b/store/tests/actionlog_test.go @@ -87,91 +87,86 @@ func testActionlog(t *testing.T, s store.Actionlogs) { req.NoError(err) req.Len(set, 5) }) + + t.Run("with paging", func(t *testing.T) { + req := require.New(t) + req.NoError(s.TruncateActionlogs(ctx)) + + set := []*actionlog.Action{ + makeNew(1, "01"), + makeNew(2, "02"), + makeNew(3, "03"), + makeNew(4, "04"), + makeNew(5, "05"), + makeNew(6, "06"), + makeNew(7, "07"), + makeNew(8, "08"), + makeNew(9, "09"), + makeNew(10, "10"), + } + + req.NoError(s.CreateActionlog(ctx, set...)) + f := actionlog.Filter{} + + // Fetch first page + f.Limit = 3 + set, f, err := store.SearchActionlogs(ctx, s, f) + req.NoError(err) + req.Len(set, 3) + req.NotNil(f.NextPage) + req.Nil(f.PrevPage) + req.Equal("10..8", stringifySetRange(set)) + + // 2nd page + f.Limit = 6 + f.PageCursor = f.NextPage + set, f, err = store.SearchActionlogs(ctx, s, f) + req.NoError(err) + req.Len(set, 6) + req.NotNil(f.NextPage) + req.NotNil(f.PrevPage) + req.Equal("7..2", stringifySetRange(set)) + + // 3rd, last page (1 item left) + f.Limit = 2 + f.PageCursor = f.NextPage + set, f, err = store.SearchActionlogs(ctx, s, f) + req.NoError(err) + req.Len(set, 1) + req.Nil(f.NextPage) + req.NotNil(f.PrevPage) + req.Equal("1", stringifySetRange(set)) + + // now, in reverse, last 3 items + f.Limit = 3 + f.PageCursor = f.PrevPage + set, f, err = store.SearchActionlogs(ctx, s, f) + req.NoError(err) + req.Len(set, 3) + req.NotNil(f.NextPage) + req.NotNil(f.PrevPage) + req.Equal("4..2", stringifySetRange(set)) + + // still in reverse, next 6 items + f.Limit = 5 + f.PageCursor = f.PrevPage + set, f, err = store.SearchActionlogs(ctx, s, f) + req.NoError(err) + req.Len(set, 5) + req.NotNil(f.NextPage) + req.NotNil(f.PrevPage) + req.Equal("9..5", stringifySetRange(set)) + + // still in reverse, last 5 items (actually, we'll only get 1) + f.Limit = 5 + f.PageCursor = f.PrevPage + set, f, err = store.SearchActionlogs(ctx, s, f) + req.NoError(err) + req.Len(set, 1) + req.Nil(f.PrevPage) + req.NotNil(f.NextPage) + req.Equal("10", stringifySetRange(set)) + }) }) - t.Run("with keyed paging", func(t *testing.T) { - req := require.New(t) - req.NoError(s.TruncateActionlogs(ctx)) - - set := []*actionlog.Action{ - makeNew(1, "01"), - makeNew(2, "02"), - makeNew(3, "03"), - makeNew(4, "04"), - makeNew(5, "05"), - makeNew(6, "06"), - makeNew(7, "07"), - makeNew(8, "08"), - makeNew(9, "09"), - makeNew(10, "10"), - } - - req.NoError(s.CreateActionlog(ctx, set...)) - f := actionlog.Filter{} - - // Fetch first page - f.Limit = 3 - set, f, err := store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 3) - req.NotNil(f.NextPage) - req.Nil(f.PrevPage) - req.Equal("10..8", stringifySetRange(set)) - - // 2nd page - f.Limit = 6 - f.PageCursor = f.NextPage - set, f, err = store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 6) - req.NotNil(f.NextPage) - req.NotNil(f.PrevPage) - req.Equal("7..2", stringifySetRange(set)) - - // 3rd, last page (1 item left) - f.Limit = 2 - f.PageCursor = f.NextPage - set, f, err = store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 1) - req.NotNil(f.NextPage) - req.NotNil(f.PrevPage) - req.Equal("1", stringifySetRange(set)) - - // try and go pass the last page - f.PageCursor = f.NextPage - set, _, err = store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 0) - - // now, in reverse, last 3 items - f.Limit = 3 - f.PageCursor = f.PrevPage - set, f, err = store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 3) - req.NotNil(f.NextPage) - req.NotNil(f.PrevPage) - req.Equal("3..1", stringifySetRange(set)) - - // still in reverse, next 6 items - f.Limit = 5 - f.PageCursor = f.PrevPage - set, f, err = store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 5) - req.NotNil(f.NextPage) - req.NotNil(f.PrevPage) - req.Equal("9..4", stringifySetRange(set)) - - // still in reverse, last 5 items (actually, we'll only get 1) - f.Limit = 5 - f.PageCursor = f.PrevPage - set, f, err = store.SearchActionlogs(ctx, s, f) - req.NoError(err) - req.Len(set, 1) - req.Nil(f.PrevPage) - req.NotNil(f.NextPage) - req.Equal("10", stringifySetRange(set)) - }) } diff --git a/store/tests/compose_records_test.go b/store/tests/compose_records_test.go index 4f74d757d..bb238bd9a 100644 --- a/store/tests/compose_records_test.go +++ b/store/tests/compose_records_test.go @@ -191,7 +191,7 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { t.Run("by Record", func(t *testing.T) { req, rr := truncAndCreate(t) rec := rr[0] - + req.NoError(s.DeleteComposeRecord(ctx, mod, rec)) _, err := s.LookupComposeRecordByID(ctx, mod, rec.ID) req.EqualError(err, store.ErrNotFound.Error()) @@ -200,7 +200,7 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { t.Run("by ID", func(t *testing.T) { req, rr := truncAndCreate(t) rec := rr[0] - + req.NoError(s.DeleteComposeRecordByID(ctx, mod, rec.ID)) _, err := s.LookupComposeRecordByID(ctx, mod, rec.ID) req.EqualError(err, store.ErrNotFound.Error()) @@ -357,15 +357,15 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { f.PageCursor = f.NextPage set, f, err = s.SearchComposeRecords(ctx, mod, f) - req.Equal("v4,;v5,", stringifyValues(set, "str1", "str3")) req.NoError(err) + req.Equal("v4,;v5,", stringifyValues(set, "str1", "str3")) req.NoError(f.Sort.Set("str3 DESC")) f.PageCursor = nil f.Limit = 1 set, f, err = s.SearchComposeRecords(ctx, mod, f) - req.Equal("three,v1", stringifyValues(set, "str3", "str1")) req.NoError(err) + req.Equal("three,v1", stringifyValues(set, "str3", "str1")) }) }) diff --git a/store/tests/main_test.go b/store/tests/main_test.go index a1400aee2..f3c6aad3e 100644 --- a/store/tests/main_test.go +++ b/store/tests/main_test.go @@ -35,7 +35,7 @@ func Test_Store(t *testing.T) { } ) - logger.Init() + logger.SetDefault(logger.MakeDebugLogger()) var ( ctx = context.Background() diff --git a/store/tests/users_test.go b/store/tests/users_test.go index 0e5b5c60a..a330cdbf0 100644 --- a/store/tests/users_test.go +++ b/store/tests/users_test.go @@ -195,7 +195,7 @@ func testUsers(t *testing.T, s store.Users) { req.Equal(prefill[0].ID, set[0].ID) }) - t.Run("with keyed paging", func(t *testing.T) { + t.Run("with paging", func(t *testing.T) { req := require.New(t) req.NoError(s.TruncateUsers(ctx)) @@ -214,7 +214,7 @@ func testUsers(t *testing.T, s store.Users) { req.NoError(s.CreateUser(ctx, set...)) f := types.UserFilter{} - f.Sort = filter.SortExprSet{&filter.SortExpr{Column: "email"}} + f.Sort.Set("email") // Fetch first page f.Limit = 3 @@ -241,16 +241,10 @@ func testUsers(t *testing.T, s store.Users) { set, f, err = store.SearchUsers(ctx, s, f) req.NoError(err) req.Len(set, 1) - req.NotNil(f.NextPage) + req.Nil(f.NextPage) req.NotNil(f.PrevPage) req.Equal("10", stringifySetRange(set)) - // try and go pass the last page - f.PageCursor = f.NextPage - set, _, err = store.SearchUsers(ctx, s, f) - req.NoError(err) - req.Len(set, 0) - // now, in reverse, last 3 items f.Limit = 3 f.PageCursor = f.PrevPage @@ -310,6 +304,34 @@ func testUsers(t *testing.T, s store.Users) { }) + t.Run("with incompatible sort", func(t *testing.T) { + req := require.New(t) + req.NoError(s.TruncateUsers(ctx)) + + set := []*types.User{ + makeNew("01"), + makeNew("02"), + makeNew("03"), + makeNew("04"), + makeNew("05"), + } + + req.NoError(s.CreateUser(ctx, set...)) + f := types.UserFilter{} + f.Sort = filter.SortExprSet{&filter.SortExpr{Column: "email", Descending: true}, &filter.SortExpr{Column: "handle", Descending: true}} + + f.Limit = 1 + set, f, err := store.SearchUsers(ctx, s, f) + req.NoError(err) + + // go to next page with different sorting + f.PageCursor = f.NextPage + f.Sort = filter.SortExprSet{&filter.SortExpr{Column: "email", Descending: true}} + set, f, err = store.SearchUsers(ctx, s, f) + req.EqualError(err, "incompatible sort") + + }) + t.Run("by role", func(t *testing.T) { t.Skip("not implemented") }) diff --git a/system/types/attachment.go b/system/types/attachment.go index dc7b62f73..48427c16a 100644 --- a/system/types/attachment.go +++ b/system/types/attachment.go @@ -28,7 +28,6 @@ type ( AttachmentFilter struct { Kind string `json:"kind,omitempty"` Filter string `json:"filter"` - Sort string `json:"sort"` // Check fn is called by store backend for each resource found function can // modify the resource and return false if store should not return it @@ -37,6 +36,7 @@ type ( Check func(*Attachment) (bool, error) `json:"-"` // Standard helpers for paging and sorting + filter.Sorting filter.Paging }