From 6120520a03ea6c34e5860c3a7de2ea2db1a44a3c Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Tue, 1 Sep 2020 07:25:34 +0200 Subject: [PATCH] Refactored (paged) fetching/searching --- compose/types/record_value.go | 2 + pkg/codegen/assets/store_rdbms.gen.go.tpl | 490 +++++++++++----------- pkg/codegen/store.go | 9 +- pkg/filter/pagination.go | 2 +- pkg/filter/sorting.go | 11 + store/compose_records.yaml | 3 + store/rdbms/actionlog.gen.go | 332 ++++++++------- store/rdbms/applications.gen.go | 371 ++++++++-------- store/rdbms/attachments.gen.go | 115 +++-- store/rdbms/compose_attachments.gen.go | 115 +++-- store/rdbms/compose_charts.gen.go | 372 ++++++++-------- store/rdbms/compose_module_fields.gen.go | 90 ++-- store/rdbms/compose_modules.gen.go | 372 ++++++++-------- store/rdbms/compose_namespaces.gen.go | 372 ++++++++-------- store/rdbms/compose_pages.gen.go | 370 ++++++++-------- store/rdbms/compose_record_value.go | 5 +- store/rdbms/compose_record_values.gen.go | 90 ++-- store/rdbms/compose_records.gen.go | 391 ++++++++--------- store/rdbms/compose_records.go | 143 +++++-- store/rdbms/credentials.gen.go | 90 ++-- store/rdbms/rbac_rules.gen.go | 88 ++-- store/rdbms/rdbms.go | 53 ++- store/rdbms/reminders.gen.go | 371 ++++++++-------- store/rdbms/role_members.gen.go | 88 ++-- store/rdbms/roles.gen.go | 373 ++++++++-------- store/rdbms/settings.gen.go | 115 +++-- store/rdbms/users.gen.go | 375 ++++++++--------- store/tests/compose_records_test.go | 113 +++++ store/tests/users_test.go | 30 +- 29 files changed, 2791 insertions(+), 2560 deletions(-) diff --git a/compose/types/record_value.go b/compose/types/record_value.go index 27dd3fa10..9004aae2e 100644 --- a/compose/types/record_value.go +++ b/compose/types/record_value.go @@ -4,6 +4,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "github.com/cortezaproject/corteza-server/pkg/rh" "time" "github.com/pkg/errors" @@ -25,6 +26,7 @@ type ( RecordValueFilter struct { RecordID []uint64 + Deleted rh.FilterState `json:"deleted"` } ) diff --git a/pkg/codegen/assets/store_rdbms.gen.go.tpl b/pkg/codegen/assets/store_rdbms.gen.go.tpl index 36b1f7cea..982053daf 100644 --- a/pkg/codegen/assets/store_rdbms.gen.go.tpl +++ b/pkg/codegen/assets/store_rdbms.gen.go.tpl @@ -17,7 +17,6 @@ import ( "github.com/cortezaproject/corteza-server/store" {{- if $.Search.EnablePaging }} "github.com/cortezaproject/corteza-server/pkg/filter" - "strings" {{- end }} {{- range $import := $.Import }} {{ normalizeImport $import }} @@ -26,6 +25,7 @@ import ( var _ = errors.Is +{{/* const ( {{- if .Create.Enable }} TriggerBefore{{ export $.Types.Singular }}Create triggerKey = "{{ unexport $.Types.Singular }}BeforeCreate" @@ -40,6 +40,7 @@ const ( TriggerBefore{{ export $.Types.Singular }}Delete triggerKey = "{{ unexport $.Types.Singular }}BeforeDelete" {{- end }} ) +*/}} {{ if $.Search.Enable }} // {{ toggleExport .Search.Export "Search" $.Types.Plural }} returns all matching rows @@ -47,286 +48,255 @@ const ( // This function calls convert{{ export $.Types.Singular }}Filter with the given // {{ $.Types.GoFilterType }} and expects to receive a working squirrel.SelectBuilder func (s Store) {{ toggleExport .Search.Export "Search" $.Types.Plural }}(ctx context.Context{{ template "extraArgsDef" . }}, f {{ $.Types.GoFilterType }}) ({{ $.Types.GoSetType }}, {{ $.Types.GoFilterType }}, error) { - var scap uint + var ( + err error + set []*{{ $.Types.GoType }} + q squirrel.SelectBuilder + ) + {{- if .RDBMS.CustomFilterConverter }} - q, err := s.convert{{ export $.Types.Singular }}Filter({{ template "extraArgsCallFirst" . }}f) + q, err = s.convert{{ export $.Types.Singular }}Filter({{ template "extraArgsCallFirst" . }}f) if err != nil { return nil, f, err } {{- else }} - q := s.{{ unexport $.Types.Plural }}SelectBuilder() + q = s.{{ unexport $.Types.Plural }}SelectBuilder() {{- end }} {{ if $.Search.EnablePaging }} - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse {{ end }} {{ if $.Search.EnableSorting }} - if err := f.Sort.Validate(s.sortable{{ export $.Types.Singular }}Columns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } - // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() - } - - // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } - } - - q = q.OrderBy(sqlSort...) + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() } {{ else if $.Search.EnablePaging }} // Sorting is disabled in definition yaml file - // {search: {enablePaging:false}} + // {search: {enableSorting:false}} // // We still need to sort the results by primary key for paging purposes - sort := filter.SortExprSet{ - {{ range $.Fields }} - {{- if or .IsPrimaryKey -}} - &filter.SortExpr{Column: {{ printf "%q" .Column }}, {{ if .SortDescending }}Descending: true, {{ end }}}, - {{- end }} + curSort := filter.SortExprSet{ + {{- range $.Fields.PrimaryKeyFields }} + &filter.SortExpr{Column: {{ printf "%q" .Column }}, {{ if .SortDescending }}Descending: !reversedCursor, {{ end }}}, {{- end }} } {{ end }} - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + {{- if $.Search.EnablePaging }} + set, err = s.{{ unexport "fetchFullPageOf" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, curSort, f.PageCursor, f.Limit, {{ if $.Search.EnableFilterCheckFn }}f.Check{{ else }}nil{{ end }},) - - var ( - set = make([]*{{ $.Types.GoType }}, 0, scap) - - {{- if not $.Search.EnablePaging }} - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *{{ $.Types.GoType }} - rows, err = s.Query(ctx, q) - ) - - if err != nil { - return err - } - - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internal{{ export $.Types.Singular }}RowScanner({{ template "extraArgsCallFirst" . }}rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - {{ if $.Search.EnableFilterCheckFn }} - if f.Check != nil { - if chk, err := f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return err - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - {{ end -}} - - set = append(set, res) - } - - return rows.Close() - } - {{ else }} - // fetches rows and scans them into {{ $.Types.GoType }} resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *{{ $.Types.GoType }} - - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) - - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internal{{ export $.Types.Singular }}RowScanner({{ template "extraArgsCallFirst" . }}rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - {{ if $.Search.EnableFilterCheckFn }} - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - {{ end -}} - - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return + if err != nil { + return err } - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collect{{ export $.Types.Singular }}CursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true } - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collect{{ export $.Types.Singular }}CursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collect{{ export $.Types.Singular }}CursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collect{{ export $.Types.Singular }}CursorValues(set[len(set)-1], curSort.Columns()...) } - {{ end -}} - ) - return set, f, s.config.ErrorHandler(fetch()) + f.PageCursor = nil + return nil + {{- else }} + set, _, _, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, q, {{ if $.Search.EnableFilterCheckFn }}f.Check{{else}}nil{{ end }}) + return err + {{ end }} + }()) } {{ end }} + +{{ if $.Search.EnablePaging }} +// {{ unexport "fetchFullPageOf" $.Types.Plural }} collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural }} ( + ctx context.Context{{ template "extraArgsDef" . }}, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*{{ $.Types.GoType }}) (bool, error), +) ([]*{{ $.Types.GoType }}, error) { + var ( + set = make([]*{{ $.Types.GoType }}, 0, DefaultSliceCapacity) + aux []*{{ $.Types.GoType }} + last *{{ $.Types.GoType }} + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + +{{ if .Fields.PrimaryKeyFields }} + // Make sure we always end our sort by primary keys + {{- range .Fields.PrimaryKeyFields }} + if sort.Get({{ printf "%q" .Column }}) == nil { + sort = append(sort, &filter.SortExpr{Column: {{ printf "%q" .Column }}}) + } + {{ end }} +{{ end }} + +{{ if .RDBMS.CustomSortConverter }} + if q, err = s.{{ unexport $.Types.Plural }}Sorter({{ template "extraArgsCallFirst" . }}q, sort); err != nil { + return nil, err + } +{{ else if .Search.EnableSorting }} + // Apply sorting expr from filter to query + if q, err = setOrderBy(q, sort, s.sortable{{ export $.Types.Singular }}Columns()...); err != nil { + return nil, err + } +{{ else if .Fields.PrimaryKeyFields }} + // Sort by primary keys by default + if q, err = setOrderBy(q, sort, {{ range .Fields.PrimaryKeyFields }}"{{ .Column }}",{{ end }}); err != nil { + return nil, err + } +{{ end }} + + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) + } + + if aux, fetched, last, err = s.{{ export "query" $.Types.Plural }}(ctx{{ template "extraArgsCall" . }}, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collect{{ export $.Types.Singular }}CursorValues(last, sort.Columns()...); cursor == nil { + break + } + } + + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } + } + + return set, nil +} +{{ end }} + + +// {{ export "query" $.Types.Plural }} queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) {{ export "query" $.Types.Plural }} ( + ctx context.Context{{ template "extraArgsDef" . }}, + q squirrel.SelectBuilder, + check func(*{{ $.Types.GoType }}) (bool, error), +) ([]*{{ $.Types.GoType }}, uint, *{{ $.Types.GoType }}, error) { + var ( + set = make([]*{{ $.Types.GoType }}, 0, DefaultSliceCapacity) + res *{{ $.Types.GoType }} + + // Query rows with + rows, err = s.Query(ctx, q) + + fetched uint + ) + + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internal{{ export $.Types.Singular }}RowScanner({{ template "extraArgsCallFirst" . }}rows) + } + + if err != nil { + return nil, 0, nil, err + } + + {{ if $.Search.EnableFilterCheckFn }} + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + {{ end }} + set = append(set, res) + } + +{{ if .RDBMS.CustomPostLoadProcessor }} + if err = s.{{ unexport $.Types.Singular }}PostLoadProcessor(ctx{{ template "extraArgsCall" . }}, set...); err != nil { + return nil, 0, nil, err + } +{{end }} + + return set, fetched, res, rows.Err() +} + + + + {{- range $lookup := $.Lookups }} // {{ toggleExport $lookup.Export "Lookup" $.Types.Singular "By" $lookup.Suffix }} {{ comment $lookup.Description true -}} func (s Store) {{ toggleExport $lookup.Export "Lookup" $.Types.Singular "By" $lookup.Suffix }}(ctx context.Context{{ template "extraArgsDef" $ }}{{- range $field := $lookup.Fields }}, {{ cc2underscore $field }} {{ ($field | $.Fields.Find).Type }}{{- end }}) (*{{ $.Types.GoType }}, error) { @@ -478,6 +448,11 @@ func (s Store) execLookup{{ $.Types.Singular }}(ctx context.Context{{ template " return } +{{ if .RDBMS.CustomPostLoadProcessor }} + if err = s.{{ unexport $.Types.Singular }}PostLoadProcessor(ctx{{ template "extraArgsCall" . }}, res); err != nil { + return nil, err + } +{{ end }} return res, nil } @@ -618,13 +593,27 @@ func (s Store) internal{{ export $.Types.Singular }}Encoder(res *{{ $.Types.GoTy {{- end }} } -{{ if $.Search.EnablePaging }} +{{ if and $.Search.EnablePaging (not $.RDBMS.CustomCursorCollector) }} +// collect{{ export $.Types.Singular }}CursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collect{{ export $.Types.Singular }}CursorValues(res *{{ $.Types.GoType }}, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + {{ range $.Fields.PrimaryKeyFields }} + pk{{ export .Column }} bool + {{ end }} + collect = func(cc ...string) { for _, c := range cc { switch c { @@ -632,9 +621,12 @@ func (s Store) collect{{ export $.Types.Singular }}CursorValues(res *{{ $.Types. {{- if or .IsSortable .IsUnique .IsPrimaryKey -}} case "{{ .Column }}": cursor.Set(c, res.{{ .Field }}, false) - {{ if .IsUnique -}} + {{ if .IsUnique -}} hasUnique = true {{ end }} + {{ if .IsPrimaryKey -}} + pk{{ export .Column }} = true + {{ end }} {{- end }} {{- end }} } @@ -643,12 +635,8 @@ func (s Store) collect{{ export $.Types.Singular }}CursorValues(res *{{ $.Types. ) collect(cc...) - if !hasUnique { - collect( - {{ range $.Fields -}} - {{ if .IsPrimaryKey }}"{{ .Column }}",{{ end }} - {{- end }} - ) + if !hasUnique || !({{ range $.Fields.PrimaryKeyFields }}pk{{ export .Column }} && {{ end }} true) { + collect({{ range $.Fields.PrimaryKeyFields }}"{{ .Column }}",{{ end }}) } return cursor diff --git a/pkg/codegen/store.go b/pkg/codegen/store.go index 294025697..fdd0399b7 100644 --- a/pkg/codegen/store.go +++ b/pkg/codegen/store.go @@ -81,9 +81,12 @@ type ( Alias string `yaml:"alias,omitempty"` Table string `yaml:"table,omitempty"` - CustomRowScanner bool `yaml:"customRowScanner"` - CustomFilterConverter bool `yaml:"customFilterConverter"` - CustomEncoder bool `yaml:"customEncoder"` + CustomRowScanner bool `yaml:"customRowScanner"` + CustomFilterConverter bool `yaml:"customFilterConverter"` + CustomSortConverter bool `yaml:"customSortConverter"` + CustomCursorCollector bool `yaml:"customCursorCollector"` + CustomPostLoadProcessor bool `yaml:"customPostLoadProcessor"` + CustomEncoder bool `yaml:"customEncoder"` } storeTypeFunctionsDef struct { diff --git a/pkg/filter/pagination.go b/pkg/filter/pagination.go index 209e83f86..01fa3caf2 100644 --- a/pkg/filter/pagination.go +++ b/pkg/filter/pagination.go @@ -72,7 +72,7 @@ func (p *PagingCursor) String() string { var o = "<" for i, key := range p.keys { - o += fmt.Sprintf("%s: %s, ", key, p.values[i]) + o += fmt.Sprintf("%s: %v, ", key, p.values[i]) } if p.Reverse { diff --git a/pkg/filter/sorting.go b/pkg/filter/sorting.go index b3678b6ed..40edd8cf3 100644 --- a/pkg/filter/sorting.go +++ b/pkg/filter/sorting.go @@ -96,6 +96,17 @@ func (set SortExprSet) Validate(cc ...string) error { return nil } +// Get returns sort expression from set if exists +func (set SortExprSet) Get(col string) *SortExpr { + for _, e := range set { + if e.Column == col { + return e + } + } + + return nil +} + // Clone returns cloned sort expression set func (set SortExprSet) Clone() (out SortExprSet) { out = make([]*SortExpr, len(set)) diff --git a/store/compose_records.yaml b/store/compose_records.yaml index 051857b0c..c5b196972 100644 --- a/store/compose_records.yaml +++ b/store/compose_records.yaml @@ -40,6 +40,9 @@ rdbms: alias: crd table: compose_record customFilterConverter: true + customSortConverter: true + customCursorCollector: true + customPostLoadProcessor: true search: export: false diff --git a/store/rdbms/actionlog.gen.go b/store/rdbms/actionlog.gen.go index a0045ebac..2b9bb1acf 100644 --- a/store/rdbms/actionlog.gen.go +++ b/store/rdbms/actionlog.gen.go @@ -17,190 +17,195 @@ import ( "github.com/cortezaproject/corteza-server/pkg/actionlog" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeActionlogCreate triggerKey = "actionlogBeforeCreate" -) - // SearchActionlogs returns all matching rows // // This function calls convertActionlogFilter with the given // actionlog.Filter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchActionlogs(ctx context.Context, f actionlog.Filter) (actionlog.ActionSet, actionlog.Filter, error) { - var scap uint - q, err := s.convertActionlogFilter(f) + var ( + err error + set []*actionlog.Action + q squirrel.SelectBuilder + ) + q, err = s.convertActionlogFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // Sorting is disabled in definition yaml file - // {search: {enablePaging:false}} + // {search: {enableSorting:false}} // // We still need to sort the results by primary key for paging purposes - sort := filter.SortExprSet{ - &filter.SortExpr{Column: "id", Descending: true}, + curSort := filter.SortExprSet{ + &filter.SortExpr{Column: "id", Descending: !reversedCursor}, } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfActionlogs(ctx, q, curSort, f.PageCursor, f.Limit, nil) + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectActionlogCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectActionlogCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfActionlogs collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfActionlogs( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*actionlog.Action) (bool, error), +) ([]*actionlog.Action, error) { var ( - set = make([]*actionlog.Action, 0, scap) - // fetches rows and scans them into actionlog.Action resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *actionlog.Action + set = make([]*actionlog.Action, 0, DefaultSliceCapacity) + aux []*actionlog.Action + last *actionlog.Action - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse - if limit > 0 { - slct = slct.Limit(uint64(limit)) + // copy of the select builder + tryQuery squirrel.SelectBuilder - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalActionlogRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectActionlogCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectActionlogCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint + err error ) - return set, f, s.config.ErrorHandler(fetch()) + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) + } + + // Sort by primary keys by default + if q, err = setOrderBy(q, sort, "id"); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) + } + + if aux, fetched, last, err = s.QueryActionlogs(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectActionlogCursorValues(last, sort.Columns()...); cursor == nil { + break + } + } + + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } + } + + return set, nil +} + +// QueryActionlogs queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryActionlogs( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*actionlog.Action) (bool, error), +) ([]*actionlog.Action, uint, *actionlog.Action, error) { + var ( + set = make([]*actionlog.Action, 0, DefaultSliceCapacity) + res *actionlog.Action + + // Query rows with + rows, err = s.Query(ctx, q) + + fetched uint + ) + + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalActionlogRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // CreateActionlog creates one or more rows in actionlog table @@ -321,28 +326,41 @@ func (s Store) internalActionlogEncoder(res *actionlog.Action) store.Payload { return s.encodeActionlog(res) } +// collectActionlogCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectActionlogCursorValues(res *actionlog.Action, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + pkId = true + } } } ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/applications.gen.go b/store/rdbms/applications.gen.go index 76b13d138..a242f9606 100644 --- a/store/rdbms/applications.gen.go +++ b/store/rdbms/applications.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/types" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeApplicationCreate triggerKey = "applicationBeforeCreate" - TriggerBeforeApplicationUpdate triggerKey = "applicationBeforeUpdate" - TriggerBeforeApplicationUpsert triggerKey = "applicationBeforeUpsert" - TriggerBeforeApplicationDelete triggerKey = "applicationBeforeDelete" -) - // SearchApplications returns all matching rows // // This function calls convertApplicationFilter with the given // types.ApplicationFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchApplications(ctx context.Context, f types.ApplicationFilter) (types.ApplicationSet, types.ApplicationFilter, error) { - var scap uint - q, err := s.convertApplicationFilter(f) + var ( + err error + set []*types.Application + q squirrel.SelectBuilder + ) + q, err = s.convertApplicationFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableApplicationColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfApplications(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectApplicationCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectApplicationCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfApplications collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfApplications( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Application) (bool, error), +) ([]*types.Application, error) { + var ( + set = make([]*types.Application, 0, DefaultSliceCapacity) + aux []*types.Application + last *types.Application + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableApplicationColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryApplications(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectApplicationCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryApplications queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryApplications( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Application) (bool, error), +) ([]*types.Application, uint, *types.Application, error) { var ( - set = make([]*types.Application, 0, scap) - // fetches rows and scans them into types.Application resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Application + set = make([]*types.Application, 0, DefaultSliceCapacity) + res *types.Application - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalApplicationRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectApplicationCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectApplicationCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalApplicationRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupApplicationByID searches for application by ID @@ -483,23 +464,41 @@ func (s Store) internalApplicationEncoder(res *types.Application) store.Payload } } +// collectApplicationCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectApplicationCursorValues(res *types.Application, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "name": cursor.Set(c, res.Name, false) + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -509,10 +508,8 @@ func (s Store) collectApplicationCursorValues(res *types.Application, cc ...stri ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/attachments.gen.go b/store/rdbms/attachments.gen.go index d01ce6e4e..4e8f02508 100644 --- a/store/rdbms/attachments.gen.go +++ b/store/rdbms/attachments.gen.go @@ -20,79 +20,78 @@ import ( var _ = errors.Is -const ( - TriggerBeforeAttachmentCreate triggerKey = "attachmentBeforeCreate" - TriggerBeforeAttachmentUpdate triggerKey = "attachmentBeforeUpdate" - TriggerBeforeAttachmentUpsert triggerKey = "attachmentBeforeUpsert" - TriggerBeforeAttachmentDelete triggerKey = "attachmentBeforeDelete" -) - // SearchAttachments returns all matching rows // // This function calls convertAttachmentFilter with the given // types.AttachmentFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchAttachments(ctx context.Context, f types.AttachmentFilter) (types.AttachmentSet, types.AttachmentFilter, error) { - var scap uint - q, err := s.convertAttachmentFilter(f) + var ( + err error + set []*types.Attachment + q squirrel.SelectBuilder + ) + q, err = s.convertAttachmentFilter(f) if err != nil { return nil, f, err } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryAttachments(ctx, q, f.Check) + return err + }()) +} + +// QueryAttachments queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryAttachments( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Attachment) (bool, error), +) ([]*types.Attachment, uint, *types.Attachment, error) { var ( - set = make([]*types.Attachment, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.Attachment - rows, err = s.Query(ctx, q) - ) + set = make([]*types.Attachment, 0, DefaultSliceCapacity) + res *types.Attachment - if err != nil { - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalAttachmentRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - if chk, err := f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return err - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalAttachmentRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupAttachmentByID searches for attachment by its ID diff --git a/store/rdbms/compose_attachments.gen.go b/store/rdbms/compose_attachments.gen.go index 8603d0d9c..d222741f8 100644 --- a/store/rdbms/compose_attachments.gen.go +++ b/store/rdbms/compose_attachments.gen.go @@ -20,79 +20,78 @@ import ( var _ = errors.Is -const ( - TriggerBeforeComposeAttachmentCreate triggerKey = "composeAttachmentBeforeCreate" - TriggerBeforeComposeAttachmentUpdate triggerKey = "composeAttachmentBeforeUpdate" - TriggerBeforeComposeAttachmentUpsert triggerKey = "composeAttachmentBeforeUpsert" - TriggerBeforeComposeAttachmentDelete triggerKey = "composeAttachmentBeforeDelete" -) - // SearchComposeAttachments returns all matching rows // // This function calls convertComposeAttachmentFilter with the given // types.AttachmentFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchComposeAttachments(ctx context.Context, f types.AttachmentFilter) (types.AttachmentSet, types.AttachmentFilter, error) { - var scap uint - q, err := s.convertComposeAttachmentFilter(f) + var ( + err error + set []*types.Attachment + q squirrel.SelectBuilder + ) + q, err = s.convertComposeAttachmentFilter(f) if err != nil { return nil, f, err } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryComposeAttachments(ctx, q, f.Check) + return err + }()) +} + +// QueryComposeAttachments queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeAttachments( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Attachment) (bool, error), +) ([]*types.Attachment, uint, *types.Attachment, error) { var ( - set = make([]*types.Attachment, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.Attachment - rows, err = s.Query(ctx, q) - ) + set = make([]*types.Attachment, 0, DefaultSliceCapacity) + res *types.Attachment - if err != nil { - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalComposeAttachmentRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - if chk, err := f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return err - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeAttachmentRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupComposeAttachmentByID searches for attachment by its ID diff --git a/store/rdbms/compose_charts.gen.go b/store/rdbms/compose_charts.gen.go index d89c246e7..51bb59bb1 100644 --- a/store/rdbms/compose_charts.gen.go +++ b/store/rdbms/compose_charts.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeComposeChartCreate triggerKey = "composeChartBeforeCreate" - TriggerBeforeComposeChartUpdate triggerKey = "composeChartBeforeUpdate" - TriggerBeforeComposeChartUpsert triggerKey = "composeChartBeforeUpsert" - TriggerBeforeComposeChartDelete triggerKey = "composeChartBeforeDelete" -) - // SearchComposeCharts returns all matching rows // // This function calls convertComposeChartFilter with the given // types.ChartFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchComposeCharts(ctx context.Context, f types.ChartFilter) (types.ChartSet, types.ChartFilter, error) { - var scap uint - q, err := s.convertComposeChartFilter(f) + var ( + err error + set []*types.Chart + q squirrel.SelectBuilder + ) + q, err = s.convertComposeChartFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableComposeChartColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfComposeCharts(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectComposeChartCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectComposeChartCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfComposeCharts collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfComposeCharts( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Chart) (bool, error), +) ([]*types.Chart, error) { + var ( + set = make([]*types.Chart, 0, DefaultSliceCapacity) + aux []*types.Chart + last *types.Chart + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableComposeChartColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryComposeCharts(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectComposeChartCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryComposeCharts queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeCharts( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Chart) (bool, error), +) ([]*types.Chart, uint, *types.Chart, error) { var ( - set = make([]*types.Chart, 0, scap) - // fetches rows and scans them into types.Chart resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Chart + set = make([]*types.Chart, 0, DefaultSliceCapacity) + res *types.Chart - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalComposeChartRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectComposeChartCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeChartCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeChartRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupComposeChartByID searches for compose chart by ID @@ -492,26 +473,45 @@ func (s Store) internalComposeChartEncoder(res *types.Chart) store.Payload { } } +// collectComposeChartCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectComposeChartCursorValues(res *types.Chart, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "handle": cursor.Set(c, res.Handle, false) hasUnique = true + case "name": cursor.Set(c, res.Name, false) + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -521,10 +521,8 @@ func (s Store) collectComposeChartCursorValues(res *types.Chart, cc ...string) * ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/compose_module_fields.gen.go b/store/rdbms/compose_module_fields.gen.go index bc2836320..dab571564 100644 --- a/store/rdbms/compose_module_fields.gen.go +++ b/store/rdbms/compose_module_fields.gen.go @@ -20,65 +20,67 @@ import ( var _ = errors.Is -const ( - TriggerBeforeComposeModuleFieldCreate triggerKey = "composeModuleFieldBeforeCreate" - TriggerBeforeComposeModuleFieldUpdate triggerKey = "composeModuleFieldBeforeUpdate" - TriggerBeforeComposeModuleFieldUpsert triggerKey = "composeModuleFieldBeforeUpsert" - TriggerBeforeComposeModuleFieldDelete triggerKey = "composeModuleFieldBeforeDelete" -) - // SearchComposeModuleFields returns all matching rows // // This function calls convertComposeModuleFieldFilter with the given // types.ModuleFieldFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchComposeModuleFields(ctx context.Context, f types.ModuleFieldFilter) (types.ModuleFieldSet, types.ModuleFieldFilter, error) { - var scap uint - q, err := s.convertComposeModuleFieldFilter(f) + var ( + err error + set []*types.ModuleField + q squirrel.SelectBuilder + ) + q, err = s.convertComposeModuleFieldFilter(f) if err != nil { return nil, f, err } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryComposeModuleFields(ctx, q, nil) + return err + }()) +} + +// QueryComposeModuleFields queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeModuleFields( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.ModuleField) (bool, error), +) ([]*types.ModuleField, uint, *types.ModuleField, error) { var ( - set = make([]*types.ModuleField, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.ModuleField - rows, err = s.Query(ctx, q) - ) + set = make([]*types.ModuleField, 0, DefaultSliceCapacity) + res *types.ModuleField - if err != nil { - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalComposeModuleFieldRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeModuleFieldRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupComposeModuleFieldByModuleIDName searches for compose module field by name (case-insensitive) diff --git a/store/rdbms/compose_modules.gen.go b/store/rdbms/compose_modules.gen.go index e299c9710..9face5d2c 100644 --- a/store/rdbms/compose_modules.gen.go +++ b/store/rdbms/compose_modules.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeComposeModuleCreate triggerKey = "composeModuleBeforeCreate" - TriggerBeforeComposeModuleUpdate triggerKey = "composeModuleBeforeUpdate" - TriggerBeforeComposeModuleUpsert triggerKey = "composeModuleBeforeUpsert" - TriggerBeforeComposeModuleDelete triggerKey = "composeModuleBeforeDelete" -) - // SearchComposeModules returns all matching rows // // This function calls convertComposeModuleFilter with the given // types.ModuleFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchComposeModules(ctx context.Context, f types.ModuleFilter) (types.ModuleSet, types.ModuleFilter, error) { - var scap uint - q, err := s.convertComposeModuleFilter(f) + var ( + err error + set []*types.Module + q squirrel.SelectBuilder + ) + q, err = s.convertComposeModuleFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableComposeModuleColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfComposeModules(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectComposeModuleCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectComposeModuleCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfComposeModules collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfComposeModules( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Module) (bool, error), +) ([]*types.Module, error) { + var ( + set = make([]*types.Module, 0, DefaultSliceCapacity) + aux []*types.Module + last *types.Module + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableComposeModuleColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryComposeModules(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectComposeModuleCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryComposeModules queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeModules( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Module) (bool, error), +) ([]*types.Module, uint, *types.Module, error) { var ( - set = make([]*types.Module, 0, scap) - // fetches rows and scans them into types.Module resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Module + set = make([]*types.Module, 0, DefaultSliceCapacity) + res *types.Module - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalComposeModuleRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectComposeModuleCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeModuleCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeModuleRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupComposeModuleByNamespaceIDHandle searches for compose module by handle (case-insensitive) @@ -500,26 +481,45 @@ func (s Store) internalComposeModuleEncoder(res *types.Module) store.Payload { } } +// collectComposeModuleCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectComposeModuleCursorValues(res *types.Module, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "handle": cursor.Set(c, res.Handle, false) hasUnique = true + case "name": cursor.Set(c, res.Name, false) + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -529,10 +529,8 @@ func (s Store) collectComposeModuleCursorValues(res *types.Module, cc ...string) ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/compose_namespaces.gen.go b/store/rdbms/compose_namespaces.gen.go index 242894fbc..9a6d68470 100644 --- a/store/rdbms/compose_namespaces.gen.go +++ b/store/rdbms/compose_namespaces.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeComposeNamespaceCreate triggerKey = "composeNamespaceBeforeCreate" - TriggerBeforeComposeNamespaceUpdate triggerKey = "composeNamespaceBeforeUpdate" - TriggerBeforeComposeNamespaceUpsert triggerKey = "composeNamespaceBeforeUpsert" - TriggerBeforeComposeNamespaceDelete triggerKey = "composeNamespaceBeforeDelete" -) - // SearchComposeNamespaces returns all matching rows // // This function calls convertComposeNamespaceFilter with the given // types.NamespaceFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchComposeNamespaces(ctx context.Context, f types.NamespaceFilter) (types.NamespaceSet, types.NamespaceFilter, error) { - var scap uint - q, err := s.convertComposeNamespaceFilter(f) + var ( + err error + set []*types.Namespace + q squirrel.SelectBuilder + ) + q, err = s.convertComposeNamespaceFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableComposeNamespaceColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfComposeNamespaces(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectComposeNamespaceCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectComposeNamespaceCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfComposeNamespaces collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfComposeNamespaces( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Namespace) (bool, error), +) ([]*types.Namespace, error) { + var ( + set = make([]*types.Namespace, 0, DefaultSliceCapacity) + aux []*types.Namespace + last *types.Namespace + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableComposeNamespaceColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryComposeNamespaces(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectComposeNamespaceCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryComposeNamespaces queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeNamespaces( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Namespace) (bool, error), +) ([]*types.Namespace, uint, *types.Namespace, error) { var ( - set = make([]*types.Namespace, 0, scap) - // fetches rows and scans them into types.Namespace resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Namespace + set = make([]*types.Namespace, 0, DefaultSliceCapacity) + res *types.Namespace - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalComposeNamespaceRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectComposeNamespaceCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeNamespaceCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeNamespaceRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupComposeNamespaceBySlug searches for namespace by slug (case-insensitive) @@ -491,26 +472,45 @@ func (s Store) internalComposeNamespaceEncoder(res *types.Namespace) store.Paylo } } +// collectComposeNamespaceCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectComposeNamespaceCursorValues(res *types.Namespace, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "name": cursor.Set(c, res.Name, false) + case "slug": cursor.Set(c, res.Slug, false) hasUnique = true + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -520,10 +520,8 @@ func (s Store) collectComposeNamespaceCursorValues(res *types.Namespace, cc ...s ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/compose_pages.gen.go b/store/rdbms/compose_pages.gen.go index b95aca24b..733815d18 100644 --- a/store/rdbms/compose_pages.gen.go +++ b/store/rdbms/compose_pages.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeComposePageCreate triggerKey = "composePageBeforeCreate" - TriggerBeforeComposePageUpdate triggerKey = "composePageBeforeUpdate" - TriggerBeforeComposePageUpsert triggerKey = "composePageBeforeUpsert" - TriggerBeforeComposePageDelete triggerKey = "composePageBeforeDelete" -) - // SearchComposePages returns all matching rows // // This function calls convertComposePageFilter with the given // types.PageFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchComposePages(ctx context.Context, f types.PageFilter) (types.PageSet, types.PageFilter, error) { - var scap uint - q, err := s.convertComposePageFilter(f) + var ( + err error + set []*types.Page + q squirrel.SelectBuilder + ) + q, err = s.convertComposePageFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableComposePageColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfComposePages(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectComposePageCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectComposePageCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfComposePages collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfComposePages( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Page) (bool, error), +) ([]*types.Page, error) { + var ( + set = make([]*types.Page, 0, DefaultSliceCapacity) + aux []*types.Page + last *types.Page + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableComposePageColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryComposePages(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectComposePageCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryComposePages queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposePages( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Page) (bool, error), +) ([]*types.Page, uint, *types.Page, error) { var ( - set = make([]*types.Page, 0, scap) - // fetches rows and scans them into types.Page resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Page + set = make([]*types.Page, 0, DefaultSliceCapacity) + res *types.Page - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalComposePageRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectComposePageCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposePageCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposePageRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupComposePageByNamespaceIDHandle searches for page by handle (case-insensitive) @@ -513,21 +494,38 @@ func (s Store) internalComposePageEncoder(res *types.Page) store.Payload { } } +// collectComposePageCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectComposePageCursorValues(res *types.Page, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -537,10 +535,8 @@ func (s Store) collectComposePageCursorValues(res *types.Page, cc ...string) *fi ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/compose_record_value.go b/store/rdbms/compose_record_value.go index c20102242..f12f814b6 100644 --- a/store/rdbms/compose_record_value.go +++ b/store/rdbms/compose_record_value.go @@ -6,10 +6,13 @@ import ( "errors" "github.com/Masterminds/squirrel" "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/rh" ) -func (s Store) convertComposeRecordValueFilter(m *types.Module, f types.RecordValueFilter) (query squirrel.SelectBuilder, err error) { +func (s Store) convertComposeRecordValueFilter(_ *types.Module, f types.RecordValueFilter) (query squirrel.SelectBuilder, err error) { + // Always filter by record IDs query = s.composeRecordValuesSelectBuilder().Where(squirrel.Eq{"crv.record_id": f.RecordID}) + query = rh.FilterNullByState(query, "crv.deleted_at", f.Deleted) return query, nil } diff --git a/store/rdbms/compose_record_values.gen.go b/store/rdbms/compose_record_values.gen.go index 5ce68c0de..6e0be5db4 100644 --- a/store/rdbms/compose_record_values.gen.go +++ b/store/rdbms/compose_record_values.gen.go @@ -20,65 +20,67 @@ import ( var _ = errors.Is -const ( - TriggerBeforeComposeRecordValueCreate triggerKey = "composeRecordValueBeforeCreate" - TriggerBeforeComposeRecordValueUpdate triggerKey = "composeRecordValueBeforeUpdate" - TriggerBeforeComposeRecordValueUpsert triggerKey = "composeRecordValueBeforeUpsert" - TriggerBeforeComposeRecordValueDelete triggerKey = "composeRecordValueBeforeDelete" -) - // searchComposeRecordValues returns all matching rows // // This function calls convertComposeRecordValueFilter with the given // types.RecordValueFilter and expects to receive a working squirrel.SelectBuilder func (s Store) searchComposeRecordValues(ctx context.Context, _mod *types.Module, f types.RecordValueFilter) (types.RecordValueSet, types.RecordValueFilter, error) { - var scap uint - q, err := s.convertComposeRecordValueFilter(_mod, f) + var ( + err error + set []*types.RecordValue + q squirrel.SelectBuilder + ) + q, err = s.convertComposeRecordValueFilter(_mod, f) if err != nil { return nil, f, err } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryComposeRecordValues(ctx, _mod, q, nil) + return err + }()) +} + +// QueryComposeRecordValues queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeRecordValues( + ctx context.Context, _mod *types.Module, + q squirrel.SelectBuilder, + check func(*types.RecordValue) (bool, error), +) ([]*types.RecordValue, uint, *types.RecordValue, error) { var ( - set = make([]*types.RecordValue, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.RecordValue - rows, err = s.Query(ctx, q) - ) + set = make([]*types.RecordValue, 0, DefaultSliceCapacity) + res *types.RecordValue - if err != nil { - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalComposeRecordValueRowScanner(_mod, rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeRecordValueRowScanner(_mod, rows) + } + + if err != nil { + return nil, 0, nil, err + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // createComposeRecordValue creates one or more rows in compose_record_value table diff --git a/store/rdbms/compose_records.gen.go b/store/rdbms/compose_records.gen.go index 25979cfc6..951607c5d 100644 --- a/store/rdbms/compose_records.gen.go +++ b/store/rdbms/compose_records.gen.go @@ -17,224 +17,208 @@ import ( "github.com/cortezaproject/corteza-server/compose/types" "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeComposeRecordCreate triggerKey = "composeRecordBeforeCreate" - TriggerBeforeComposeRecordUpdate triggerKey = "composeRecordBeforeUpdate" - TriggerBeforeComposeRecordUpsert triggerKey = "composeRecordBeforeUpsert" - TriggerBeforeComposeRecordDelete triggerKey = "composeRecordBeforeDelete" -) - // searchComposeRecords returns all matching rows // // This function calls convertComposeRecordFilter with the given // types.RecordFilter and expects to receive a working squirrel.SelectBuilder func (s Store) searchComposeRecords(ctx context.Context, _mod *types.Module, f types.RecordFilter) (types.RecordSet, types.RecordFilter, error) { - var scap uint - q, err := s.convertComposeRecordFilter(_mod, f) + var ( + err error + set []*types.Record + q squirrel.SelectBuilder + ) + q, err = s.convertComposeRecordFilter(_mod, f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableComposeRecordColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() } - // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfComposeRecords(ctx, _mod, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err } - q = q.OrderBy(sqlSort...) - } + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectComposeRecordCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } - if scap == 0 { - scap = DefaultSliceCapacity - } + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectComposeRecordCursorValues(set[len(set)-1], curSort.Columns()...) + } + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfComposeRecords collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfComposeRecords( + ctx context.Context, _mod *types.Module, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Record) (bool, error), +) ([]*types.Record, error) { var ( - set = make([]*types.Record, 0, scap) - // fetches rows and scans them into types.Record resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Record + set = make([]*types.Record, 0, DefaultSliceCapacity) + aux []*types.Record + last *types.Record - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse - if limit > 0 { - slct = slct.Limit(uint64(limit)) + // copy of the select builder + tryQuery squirrel.SelectBuilder - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalComposeRecordRowScanner(_mod, rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectComposeRecordCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectComposeRecordCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint + err error ) - return set, f, s.config.ErrorHandler(fetch()) + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) + } + + if q, err = s.composeRecordsSorter(_mod, q, sort); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) + } + + if aux, fetched, last, err = s.QueryComposeRecords(ctx, _mod, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectComposeRecordCursorValues(last, sort.Columns()...); cursor == nil { + break + } + } + + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } + } + + return set, nil +} + +// QueryComposeRecords queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryComposeRecords( + ctx context.Context, _mod *types.Module, + q squirrel.SelectBuilder, + check func(*types.Record) (bool, error), +) ([]*types.Record, uint, *types.Record, error) { + var ( + set = make([]*types.Record, 0, DefaultSliceCapacity) + res *types.Record + + // Query rows with + rows, err = s.Query(ctx, q) + + fetched uint + ) + + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalComposeRecordRowScanner(_mod, rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + if err = s.composeRecordPostLoadProcessor(ctx, _mod, set...); err != nil { + return nil, 0, nil, err + } + + return set, fetched, res, rows.Err() } // lookupComposeRecordByID searches for compose record by ID @@ -350,6 +334,10 @@ func (s Store) execLookupComposeRecord(ctx context.Context, _mod *types.Module, return } + if err = s.composeRecordPostLoadProcessor(ctx, _mod, res); err != nil { + return nil, err + } + return res, nil } @@ -487,39 +475,6 @@ func (s Store) internalComposeRecordEncoder(res *types.Record) store.Payload { } } -func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string) *filter.PagingCursor { - var ( - cursor = &filter.PagingCursor{} - - hasUnique bool - - collect = func(cc ...string) { - for _, c := range cc { - switch c { - case "id": - cursor.Set(c, res.ID, false) - case "created_at": - cursor.Set(c, res.CreatedAt, false) - case "updated_at": - cursor.Set(c, res.UpdatedAt, false) - case "deleted_at": - cursor.Set(c, res.DeletedAt, false) - - } - } - } - ) - - collect(cc...) - if !hasUnique { - collect( - "id", - ) - } - - return cursor -} - func (s *Store) checkComposeRecordConstraints(ctx context.Context, _mod *types.Module, res *types.Record) error { return nil diff --git a/store/rdbms/compose_records.go b/store/rdbms/compose_records.go index d001b57bd..5ec16cd8a 100644 --- a/store/rdbms/compose_records.go +++ b/store/rdbms/compose_records.go @@ -5,12 +5,19 @@ import ( "fmt" "github.com/Masterminds/squirrel" "github.com/cortezaproject/corteza-server/compose/types" + "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/pkg/ql" "github.com/cortezaproject/corteza-server/pkg/rh" + "github.com/cortezaproject/corteza-server/pkg/slice" "github.com/cortezaproject/corteza-server/store" "strings" ) +const ( + composeRecordValueAliasPfx = "rv_" + composeRecordValueJoinTpl = "compose_record_value AS {alias} ON ({alias}.record_id = crd.id AND {alias}.name = '{field}' AND {alias}.deleted_at IS NULL)" +) + // @todo support for partitioned records (records are partitioned into multiple record tables // in this case, values are no longer separated into record_value (key-value) table but encoded as JSON // @todo support for partitioned record with (optional) physical columns for record values @@ -35,11 +42,6 @@ func (s Store) LookupComposeRecordByID(ctx context.Context, _ *types.Module, id return } - res.Values, _, err = s.searchComposeRecordValues(ctx, nil, types.RecordValueFilter{RecordID: []uint64{id}}) - if err != nil { - return nil, err - } - return } @@ -222,10 +224,10 @@ func (s Store) convertComposeRecordFilter(m *types.Module, f types.RecordFilter) } if !alreadyJoined(i.Value) { - query = query.LeftJoin(fmt.Sprintf( - "compose_record_value AS rv_%s ON (rv_%s.record_id = crd.id AND rv_%s.name = ? AND rv_%s.deleted_at IS NULL)", - i.Value, i.Value, i.Value, i.Value, - ), i.Value) + join := composeRecordValueJoinTpl + join = strings.ReplaceAll(join, "{alias}", composeRecordValueAliasPfx+i.Value) + join = strings.ReplaceAll(join, "{field}", i.Value) + query = query.LeftJoin(join) } return s.FieldToColumnTypeCaster(m.Fields.FindByName(i.Value), i) @@ -263,30 +265,113 @@ func (s Store) convertComposeRecordFilter(m *types.Module, f types.RecordFilter) } } - // @todo refactor - //if f.Sort != "" { - // var ( - // // Sort parser - // sp = ql.NewParser() - // - // // Sort columns - // sc ql.Columns - // ) - // - // // Resolve all identifiers found in sort - // // into their table/column counterparts - // sp.OnIdent = identResolver - // - // if sc, err = sp.ParseColumns(f.Sort); err != nil { - // return - // } - // - // query = query.OrderBy(sc.Strings()...) - //} + if len(f.Sort) > 0 { + var ( + // Sort parser + sp = ql.NewParser() + ) + + // Resolve all identifiers found in sort + // into their table/column counterparts + sp.OnIdent = identResolver + + if _, err = sp.ParseColumns(f.Sort.String()); err != nil { + return + } + } return } +func (s Store) composeRecordPostLoadProcessor(ctx context.Context, m *types.Module, set ...*types.Record) (err error) { + if len(set) > 0 { + // Load all related record values and append them to each record + var ( + rvs types.RecordValueSet + ) + rvs, _, err = s.searchComposeRecordValues(ctx, nil, types.RecordValueFilter{ + RecordID: types.RecordSet(set).IDs(), + Deleted: rh.FilterStateInclusive, + }) + if err != nil { + return + } + + for r := range set { + set[r].Values = rvs.FilterByRecordID(set[r].ID) + } + } + + return nil +} + +func (s Store) composeRecordsSorter(m *types.Module, q squirrel.SelectBuilder, sort filter.SortExprSet) (squirrel.SelectBuilder, error) { + var ( + sortable = slice.ToStringBoolMap(s.sortableComposeRecordColumns()) + sqlSort = make([]string, len(sort)) + ) + + for i, c := range sort { + if sortable[c.Column] { + sqlSort[i] = sort[i].Column + } else if m.Fields.HasName(c.Column) { + //sqlSort[i] = fmt.Sprintf("%s%s.value", composeRecordValueAliasPfx, sort[i].Column) + sqlSort[i] = fmt.Sprintf("COALESCE(%s%s.value, '')", composeRecordValueAliasPfx, sort[i].Column) + } else { + return q, fmt.Errorf("could not sort by unknown column: %s", c.Column) + } + + if sort[i].Descending { + sqlSort[i] += " DESC" + } + } + + return q.OrderBy(sqlSort...), nil +} + +// Custom implementation for collecting cursor values from compose records AND it's values! +func (s Store) collectComposeRecordCursorValues(res *types.Record, cc ...string) *filter.PagingCursor { + var ( + cursor = &filter.PagingCursor{} + + hasUnique bool + pkID bool + + collect = func(cc ...string) { + for _, c := range cc { + switch c { + case "crd.id": + cursor.Set(c, res.ID, false) + pkID = true + case "crd.created_at": + cursor.Set(c, res.CreatedAt, false) + case "crd.updated_at": + cursor.Set(c, res.UpdatedAt, false) + case "crd.deleted_at": + cursor.Set(c, res.DeletedAt, false) + default: + if rv := res.Values.Get(c, 0); rv != nil { + cursor.Set(fmt.Sprintf("%s%s.value", composeRecordValueAliasPfx, c), rv.Value, false) + } else { + cursor.Set(fmt.Sprintf("%s%s.value", composeRecordValueAliasPfx, c), nil, false) + } + + //cursor.Set(fmt.Sprintf("COALESCE(%s%s.value, '')", composeRecordValueAliasPfx, c), value, false) + } + } + } + ) + + collect(cc...) + if !hasUnique || !pkID { + collect( + "crd.id", + ) + } + + return cursor +} + //// Checks if field name is "real column", reformats it and returns func isRealRecordCol(name string) (string, bool) { switch name { diff --git a/store/rdbms/credentials.gen.go b/store/rdbms/credentials.gen.go index fbb432e2e..16b235ded 100644 --- a/store/rdbms/credentials.gen.go +++ b/store/rdbms/credentials.gen.go @@ -20,65 +20,67 @@ import ( var _ = errors.Is -const ( - TriggerBeforeCredentialsCreate triggerKey = "credentialsBeforeCreate" - TriggerBeforeCredentialsUpdate triggerKey = "credentialsBeforeUpdate" - TriggerBeforeCredentialsUpsert triggerKey = "credentialsBeforeUpsert" - TriggerBeforeCredentialsDelete triggerKey = "credentialsBeforeDelete" -) - // SearchCredentials returns all matching rows // // This function calls convertCredentialsFilter with the given // types.CredentialsFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchCredentials(ctx context.Context, f types.CredentialsFilter) (types.CredentialsSet, types.CredentialsFilter, error) { - var scap uint - q, err := s.convertCredentialsFilter(f) + var ( + err error + set []*types.Credentials + q squirrel.SelectBuilder + ) + q, err = s.convertCredentialsFilter(f) if err != nil { return nil, f, err } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryCredentials(ctx, q, nil) + return err + }()) +} + +// QueryCredentials queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryCredentials( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Credentials) (bool, error), +) ([]*types.Credentials, uint, *types.Credentials, error) { var ( - set = make([]*types.Credentials, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.Credentials - rows, err = s.Query(ctx, q) - ) + set = make([]*types.Credentials, 0, DefaultSliceCapacity) + res *types.Credentials - if err != nil { - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalCredentialsRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalCredentialsRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupCredentialsByID searches for credentials by ID diff --git a/store/rdbms/rbac_rules.gen.go b/store/rdbms/rbac_rules.gen.go index 1c48e6010..1777c9858 100644 --- a/store/rdbms/rbac_rules.gen.go +++ b/store/rdbms/rbac_rules.gen.go @@ -20,62 +20,64 @@ import ( var _ = errors.Is -const ( - TriggerBeforeRbacRuleCreate triggerKey = "rbacRuleBeforeCreate" - TriggerBeforeRbacRuleUpdate triggerKey = "rbacRuleBeforeUpdate" - TriggerBeforeRbacRuleUpsert triggerKey = "rbacRuleBeforeUpsert" - TriggerBeforeRbacRuleDelete triggerKey = "rbacRuleBeforeDelete" -) - // SearchRbacRules returns all matching rows // // This function calls convertRbacRuleFilter with the given // permissions.RuleFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchRbacRules(ctx context.Context, f permissions.RuleFilter) (permissions.RuleSet, permissions.RuleFilter, error) { - var scap uint - q := s.rbacRulesSelectBuilder() - - if scap == 0 { - scap = DefaultSliceCapacity - } - var ( - set = make([]*permissions.Rule, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *permissions.Rule - rows, err = s.Query(ctx, q) - ) + err error + set []*permissions.Rule + q squirrel.SelectBuilder + ) + q = s.rbacRulesSelectBuilder() - if err != nil { - return err - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryRbacRules(ctx, q, nil) + return err - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalRbacRuleRowScanner(rows) - } + }()) +} - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } +// QueryRbacRules queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryRbacRules( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*permissions.Rule) (bool, error), +) ([]*permissions.Rule, uint, *permissions.Rule, error) { + var ( + set = make([]*permissions.Rule, 0, DefaultSliceCapacity) + res *permissions.Rule - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - // If check function is set, call it and act accordingly - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalRbacRuleRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // CreateRbacRule creates one or more rows in rbac_rules table diff --git a/store/rdbms/rdbms.go b/store/rdbms/rdbms.go index 1c54f935c..fe033deed 100644 --- a/store/rdbms/rdbms.go +++ b/store/rdbms/rdbms.go @@ -5,7 +5,9 @@ import ( "database/sql" "fmt" "github.com/Masterminds/squirrel" + "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/pkg/ql" + "github.com/cortezaproject/corteza-server/pkg/slice" "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/store/rdbms/ddl" "github.com/jmoiron/sqlx" @@ -134,8 +136,8 @@ const ( DefaultSliceCapacity = 1000 - MinRefetchLimit = 10 - MaxRefetches = 100 + MinEnsureFetchLimit = 10 + MaxRefetches = 100 ) func New(ctx context.Context, cfg *Config) (*Store, error) { @@ -209,6 +211,12 @@ func (s Store) Query(ctx context.Context, q squirrel.SelectBuilder) (*sql.Rows, return nil, fmt.Errorf("could not build query: %w", err) } + //println("############################################################") + //println(query) + //println("############################################################") + //fmt.Printf("%v\n", args) + //println("############################################################") + return s.db.QueryContext(ctx, query, args...) } @@ -229,6 +237,12 @@ func (s Store) Exec(ctx context.Context, sqlizer squirrel.Sqlizer) error { return err } + //println("############################################################") + //println(query) + //println("############################################################") + //fmt.Printf("%v\n", args) + //println("############################################################") + _, err = s.db.ExecContext(ctx, query, args...) return err } @@ -412,6 +426,41 @@ func tx(ctx context.Context, dbCandidate interface{}, cfg *Config, txOpt *sql.Tx } } +func setCursorCond(q squirrel.SelectBuilder, cursor *filter.PagingCursor) squirrel.SelectBuilder { + if cursor != nil && len(cursor.Keys()) > 0 { + const cursorTpl = `(%s) %s (?%s)` + op := ">" + if cursor.Reverse { + op = "<" + } + + pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) + q = q.Where(pred, cursor.Values()...) + } + + return q +} + +func setOrderBy(q squirrel.SelectBuilder, sort filter.SortExprSet, ss ...string) (squirrel.SelectBuilder, error) { + var ( + sortable = slice.ToStringBoolMap(ss) + sqlSort = make([]string, len(sort)) + ) + for i, c := range sort { + if sortable[c.Column] { + sqlSort[i] = sort[i].Column + } else { + return q, fmt.Errorf("could not sort by unknown column: %s", c.Column) + } + + if sort[i].Descending { + sqlSort[i] += " DESC" + } + } + + return q.OrderBy(sqlSort...), nil +} + // TxNoRetry - Transaction retry handler // // Only returns false so transactions will never retry diff --git a/store/rdbms/reminders.gen.go b/store/rdbms/reminders.gen.go index d8c65e61b..6e676802f 100644 --- a/store/rdbms/reminders.gen.go +++ b/store/rdbms/reminders.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/types" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeReminderCreate triggerKey = "reminderBeforeCreate" - TriggerBeforeReminderUpdate triggerKey = "reminderBeforeUpdate" - TriggerBeforeReminderUpsert triggerKey = "reminderBeforeUpsert" - TriggerBeforeReminderDelete triggerKey = "reminderBeforeDelete" -) - // SearchReminders returns all matching rows // // This function calls convertReminderFilter with the given // types.ReminderFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchReminders(ctx context.Context, f types.ReminderFilter) (types.ReminderSet, types.ReminderFilter, error) { - var scap uint - q, err := s.convertReminderFilter(f) + var ( + err error + set []*types.Reminder + q squirrel.SelectBuilder + ) + q, err = s.convertReminderFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableReminderColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfReminders(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectReminderCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectReminderCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfReminders collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfReminders( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Reminder) (bool, error), +) ([]*types.Reminder, error) { + var ( + set = make([]*types.Reminder, 0, DefaultSliceCapacity) + aux []*types.Reminder + last *types.Reminder + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableReminderColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryReminders(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectReminderCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryReminders queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryReminders( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Reminder) (bool, error), +) ([]*types.Reminder, uint, *types.Reminder, error) { var ( - set = make([]*types.Reminder, 0, scap) - // fetches rows and scans them into types.Reminder resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Reminder + set = make([]*types.Reminder, 0, DefaultSliceCapacity) + res *types.Reminder - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalReminderRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectReminderCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectReminderCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalReminderRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupReminderByID searches for reminder by its ID @@ -498,23 +479,41 @@ func (s Store) internalReminderEncoder(res *types.Reminder) store.Payload { } } +// collectReminderCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectReminderCursorValues(res *types.Reminder, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "remind_at": cursor.Set(c, res.RemindAt, false) + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -524,10 +523,8 @@ func (s Store) collectReminderCursorValues(res *types.Reminder, cc ...string) *f ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/role_members.gen.go b/store/rdbms/role_members.gen.go index bd54ef37f..a691a391b 100644 --- a/store/rdbms/role_members.gen.go +++ b/store/rdbms/role_members.gen.go @@ -20,62 +20,64 @@ import ( var _ = errors.Is -const ( - TriggerBeforeRoleMemberCreate triggerKey = "roleMemberBeforeCreate" - TriggerBeforeRoleMemberUpdate triggerKey = "roleMemberBeforeUpdate" - TriggerBeforeRoleMemberUpsert triggerKey = "roleMemberBeforeUpsert" - TriggerBeforeRoleMemberDelete triggerKey = "roleMemberBeforeDelete" -) - // SearchRoleMembers returns all matching rows // // This function calls convertRoleMemberFilter with the given // types.RoleMemberFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchRoleMembers(ctx context.Context, f types.RoleMemberFilter) (types.RoleMemberSet, types.RoleMemberFilter, error) { - var scap uint - q := s.roleMembersSelectBuilder() - - if scap == 0 { - scap = DefaultSliceCapacity - } - var ( - set = make([]*types.RoleMember, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.RoleMember - rows, err = s.Query(ctx, q) - ) + err error + set []*types.RoleMember + q squirrel.SelectBuilder + ) + q = s.roleMembersSelectBuilder() - if err != nil { - return err - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QueryRoleMembers(ctx, q, nil) + return err - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalRoleMemberRowScanner(rows) - } + }()) +} - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } +// QueryRoleMembers queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryRoleMembers( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.RoleMember) (bool, error), +) ([]*types.RoleMember, uint, *types.RoleMember, error) { + var ( + set = make([]*types.RoleMember, 0, DefaultSliceCapacity) + res *types.RoleMember - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - // If check function is set, call it and act accordingly - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalRoleMemberRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // CreateRoleMember creates one or more rows in role_members table diff --git a/store/rdbms/roles.gen.go b/store/rdbms/roles.gen.go index b6383fd08..4a18b4469 100644 --- a/store/rdbms/roles.gen.go +++ b/store/rdbms/roles.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/types" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeRoleCreate triggerKey = "roleBeforeCreate" - TriggerBeforeRoleUpdate triggerKey = "roleBeforeUpdate" - TriggerBeforeRoleUpsert triggerKey = "roleBeforeUpsert" - TriggerBeforeRoleDelete triggerKey = "roleBeforeDelete" -) - // SearchRoles returns all matching rows // // This function calls convertRoleFilter with the given // types.RoleFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchRoles(ctx context.Context, f types.RoleFilter) (types.RoleSet, types.RoleFilter, error) { - var scap uint - q, err := s.convertRoleFilter(f) + var ( + err error + set []*types.Role + q squirrel.SelectBuilder + ) + q, err = s.convertRoleFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableRoleColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfRoles(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectRoleCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectRoleCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfRoles collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfRoles( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.Role) (bool, error), +) ([]*types.Role, error) { + var ( + set = make([]*types.Role, 0, DefaultSliceCapacity) + aux []*types.Role + last *types.Role + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableRoleColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryRoles(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectRoleCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryRoles queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryRoles( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.Role) (bool, error), +) ([]*types.Role, uint, *types.Role, error) { var ( - set = make([]*types.Role, 0, scap) - // fetches rows and scans them into types.Role resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.Role + set = make([]*types.Role, 0, DefaultSliceCapacity) + res *types.Role - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalRoleRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectRoleCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectRoleCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalRoleRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupRoleByID searches for role by ID @@ -506,28 +487,48 @@ func (s Store) internalRoleEncoder(res *types.Role) store.Payload { } } +// collectRoleCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectRoleCursorValues(res *types.Role, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "name": cursor.Set(c, res.Name, false) + case "handle": cursor.Set(c, res.Handle, false) hasUnique = true + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "archived_at": cursor.Set(c, res.ArchivedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -537,10 +538,8 @@ func (s Store) collectRoleCursorValues(res *types.Role, cc ...string) *filter.Pa ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/rdbms/settings.gen.go b/store/rdbms/settings.gen.go index 5fb2ce801..de8bf5483 100644 --- a/store/rdbms/settings.gen.go +++ b/store/rdbms/settings.gen.go @@ -20,79 +20,78 @@ import ( var _ = errors.Is -const ( - TriggerBeforeSettingCreate triggerKey = "settingBeforeCreate" - TriggerBeforeSettingUpdate triggerKey = "settingBeforeUpdate" - TriggerBeforeSettingUpsert triggerKey = "settingBeforeUpsert" - TriggerBeforeSettingDelete triggerKey = "settingBeforeDelete" -) - // SearchSettings returns all matching rows // // This function calls convertSettingFilter with the given // types.SettingsFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchSettings(ctx context.Context, f types.SettingsFilter) (types.SettingValueSet, types.SettingsFilter, error) { - var scap uint - q, err := s.convertSettingFilter(f) + var ( + err error + set []*types.SettingValue + q squirrel.SelectBuilder + ) + q, err = s.convertSettingFilter(f) if err != nil { return nil, f, err } - if scap == 0 { - scap = DefaultSliceCapacity - } + return set, f, s.config.ErrorHandler(func() error { + set, _, _, err = s.QuerySettings(ctx, q, f.Check) + return err + }()) +} + +// QuerySettings queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QuerySettings( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.SettingValue) (bool, error), +) ([]*types.SettingValue, uint, *types.SettingValue, error) { var ( - set = make([]*types.SettingValue, 0, scap) - // Paging is disabled in definition yaml file - // {search: {enablePaging:false}} and this allows - // a much simpler row fetching logic - fetch = func() error { - var ( - res *types.SettingValue - rows, err = s.Query(ctx, q) - ) + set = make([]*types.SettingValue, 0, DefaultSliceCapacity) + res *types.SettingValue - if err != nil { - return err - } + // Query rows with + rows, err = s.Query(ctx, q) - for rows.Next() { - if err = rows.Err(); err == nil { - res, err = s.internalSettingRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - if chk, err := f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return err - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - } - - return rows.Close() - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalSettingRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupSettingByNameOwnedBy searches for settings by name and owner diff --git a/store/rdbms/users.gen.go b/store/rdbms/users.gen.go index 135d1a287..4f825dbc9 100644 --- a/store/rdbms/users.gen.go +++ b/store/rdbms/users.gen.go @@ -17,224 +17,205 @@ import ( "github.com/cortezaproject/corteza-server/pkg/filter" "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/types" - "strings" ) var _ = errors.Is -const ( - TriggerBeforeUserCreate triggerKey = "userBeforeCreate" - TriggerBeforeUserUpdate triggerKey = "userBeforeUpdate" - TriggerBeforeUserUpsert triggerKey = "userBeforeUpsert" - TriggerBeforeUserDelete triggerKey = "userBeforeDelete" -) - // SearchUsers returns all matching rows // // This function calls convertUserFilter with the given // types.UserFilter and expects to receive a working squirrel.SelectBuilder func (s Store) SearchUsers(ctx context.Context, f types.UserFilter) (types.UserSet, types.UserFilter, error) { - var scap uint - q, err := s.convertUserFilter(f) + var ( + err error + set []*types.User + q squirrel.SelectBuilder + ) + q, err = s.convertUserFilter(f) if err != nil { return nil, f, err } - scap = f.Limit - // Cleanup anything we've accidentally received... f.PrevPage, f.NextPage = nil, nil // When cursor for a previous page is used it's marked as reversed // This tells us to flip the descending flag on all used sort keys - reverseCursor := f.PageCursor != nil && f.PageCursor.Reverse - - if err := f.Sort.Validate(s.sortableUserColumns()...); err != nil { - return nil, f, fmt.Errorf("could not validate sort: %v", err) - } + reversedCursor := f.PageCursor != nil && f.PageCursor.Reverse // If paging with reverse cursor, change the sorting // direction for all columns we're sorting by - sort := f.Sort.Clone() - if reverseCursor { - sort.Reverse() + curSort := f.Sort.Clone() + if reversedCursor { + curSort.Reverse() + } + + return set, f, s.config.ErrorHandler(func() error { + set, err = s.fetchFullPageOfUsers(ctx, q, curSort, f.PageCursor, f.Limit, f.Check) + + if err != nil { + return err + } + + if f.Limit > 0 && len(set) > 0 { + if f.PageCursor != nil && (!f.PageCursor.Reverse || uint(len(set)) == f.Limit) { + f.PrevPage = s.collectUserCursorValues(set[0], curSort.Columns()...) + f.PrevPage.Reverse = true + } + + // Less items fetched then requested by page-limit + // not very likely there's another page + f.NextPage = s.collectUserCursorValues(set[len(set)-1], curSort.Columns()...) + } + + f.PageCursor = nil + return nil + }()) +} + +// fetchFullPageOfUsers collects all requested results. +// +// Function applies: +// - cursor conditions (where ...) +// - sorting rules (order by ...) +// - limit +// +// Main responsibility of this function is to perform additional sequential queries in case when not enough results +// are collected due to failed check on a specific row (by check fn). Function then moves cursor to the last item fetched +func (s Store) fetchFullPageOfUsers( + ctx context.Context, + q squirrel.SelectBuilder, + sort filter.SortExprSet, + cursor *filter.PagingCursor, + limit uint, + check func(*types.User) (bool, error), +) ([]*types.User, error) { + var ( + set = make([]*types.User, 0, DefaultSliceCapacity) + aux []*types.User + last *types.User + + // When cursor for a previous page is used it's marked as reversed + // This tells us to flip the descending flag on all used sort keys + reversedCursor = cursor != nil && cursor.Reverse + + // copy of the select builder + tryQuery squirrel.SelectBuilder + + fetched uint + err error + ) + + // Make sure we always end our sort by primary keys + if sort.Get("id") == nil { + sort = append(sort, &filter.SortExpr{Column: "id"}) } // Apply sorting expr from filter to query - if len(sort) > 0 { - sqlSort := make([]string, len(sort)) - for i := range sort { - sqlSort[i] = sort[i].Column - if sort[i].Descending { - sqlSort[i] += " DESC" - } + if q, err = setOrderBy(q, sort, s.sortableUserColumns()...); err != nil { + return nil, err + } + + for try := 0; try < MaxRefetches; try++ { + tryQuery = setCursorCond(q, cursor) + if limit > 0 { + tryQuery = tryQuery.Limit(uint64(limit)) } - q = q.OrderBy(sqlSort...) + if aux, fetched, last, err = s.QueryUsers(ctx, tryQuery, check); err != nil { + return nil, err + } + + if limit > 0 && uint(len(aux)) >= limit { + // we should use only as much as requested + set = append(set, aux[0:limit]...) + break + } else { + set = append(set, aux...) + } + + // if limit is not set or we've already collected enough items + // we can break the loop right away + if limit == 0 || fetched == 0 || fetched < limit { + break + } + + // In case limit is set very low and we've missed records in the first fetch, + // make sure next fetch limit is a bit higher + if limit < MinEnsureFetchLimit { + limit = MinEnsureFetchLimit + } + + // @todo improve strategy for collecting next page with lower limit + + // Point cursor to the last fetched element + if cursor = s.collectUserCursorValues(last, sort.Columns()...); cursor == nil { + break + } } - if scap == 0 { - scap = DefaultSliceCapacity + if reversedCursor { + // Cursor for previous page was used + // Fetched set needs to be reverseCursor because we've forced a descending order to + // get the previous page + for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { + set[i], set[j] = set[j], set[i] + } } + return set, nil +} + +// QueryUsers queries the database, converts and checks each row and +// returns collected set +// +// Fn also returns total number of fetched items and last fetched item so that the caller can construct cursor +// for next page of results +func (s Store) QueryUsers( + ctx context.Context, + q squirrel.SelectBuilder, + check func(*types.User) (bool, error), +) ([]*types.User, uint, *types.User, error) { var ( - set = make([]*types.User, 0, scap) - // fetches rows and scans them into types.User resource this is then passed to Check function on filter - // to help determine if fetched resource fits or not - // - // Note that limit is passed explicitly and is not necessarily equal to filter's limit. We want - // to keep that value intact. - // - // The value for cursor is used and set directly from/to the filter! - // - // It returns total number of fetched pages and modifies PageCursor value for paging - fetchPage = func(cursor *filter.PagingCursor, limit uint) (fetched uint, err error) { - var ( - res *types.User + set = make([]*types.User, 0, DefaultSliceCapacity) + res *types.User - // Make a copy of the select query builder so that we don't change - // the original query - slct = q.Options() - ) + // Query rows with + rows, err = s.Query(ctx, q) - if limit > 0 { - slct = slct.Limit(uint64(limit)) - - if cursor != nil && len(cursor.Keys()) > 0 { - const cursorTpl = `(%s) %s (?%s)` - op := ">" - if cursor.Reverse { - op = "<" - } - - pred := fmt.Sprintf(cursorTpl, strings.Join(cursor.Keys(), ", "), op, strings.Repeat(", ?", len(cursor.Keys())-1)) - slct = slct.Where(pred, cursor.Values()...) - } - } - - rows, err := s.Query(ctx, slct) - if err != nil { - return - } - - for rows.Next() { - fetched++ - - if rows.Err() == nil { - res, err = s.internalUserRowScanner(rows) - } - - if err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return - } - - // If check function is set, call it and act accordingly - - if f.Check != nil { - var chk bool - if chk, err = f.Check(res); err != nil { - if cerr := rows.Close(); cerr != nil { - err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) - } - - return - } else if !chk { - // did not pass the check - // go with the next row - continue - } - } - set = append(set, res) - - if f.Limit > 0 { - if uint(len(set)) >= f.Limit { - // make sure we do not fetch more than requested! - break - } - } - } - - err = rows.Close() - return - } - - fetch = func() error { - var ( - // how many items were actually fetched - fetched uint - - // starting offset & limit are from filter arg - // note that this will have to be improved with key-based pagination - limit = f.Limit - - // Copy cursor value - // - // This is where we'll start fetching and this value will be overwritten when - // results come back - cursor = f.PageCursor - - lastSetFull bool - ) - - for refetch := 0; refetch < MaxRefetches; refetch++ { - if fetched, err = fetchPage(cursor, limit); err != nil { - return err - } - - // if limit is not set or we've already collected enough items - // we can break the loop right away - if limit == 0 || fetched == 0 || fetched < limit { - break - } - - if uint(len(set)) >= f.Limit { - // we should return as much as requested - set = set[0:f.Limit] - lastSetFull = true - break - } - - // In case limit is set very low and we've missed records in the first fetch, - // make sure next fetch limit is a bit higher - if limit < MinRefetchLimit { - limit = MinRefetchLimit - } - - // @todo it might be good to implement different kind of strategies - // (beyond min-refetch-limit above) that can adjust limit on - // retry to more optimal number - } - - if reverseCursor { - // Cursor for previous page was used - // Fetched set needs to be reverseCursor because we've forced a descending order to - // get the previus page - for i, j := 0, len(set)-1; i < j; i, j = i+1, j-1 { - set[i], set[j] = set[j], set[i] - } - } - - if f.Limit > 0 && len(set) > 0 { - if f.PageCursor != nil && (!f.PageCursor.Reverse || lastSetFull) { - f.PrevPage = s.collectUserCursorValues(set[0], sort.Columns()...) - f.PrevPage.Reverse = true - } - - // Less items fetched then requested by page-limit - // not very likely there's another page - f.NextPage = s.collectUserCursorValues(set[len(set)-1], sort.Columns()...) - } - - f.PageCursor = nil - return nil - } + fetched uint ) - return set, f, s.config.ErrorHandler(fetch()) + if err != nil { + return nil, 0, nil, err + } + + defer rows.Close() + for rows.Next() { + fetched++ + if err = rows.Err(); err == nil { + res, err = s.internalUserRowScanner(rows) + } + + if err != nil { + return nil, 0, nil, err + } + + // If check function is set, call it and act accordingly + if check != nil { + if chk, err := check(res); err != nil { + return nil, 0, nil, err + } else if !chk { + // did not pass the check + // go with the next row + continue + } + } + + set = append(set, res) + } + + return set, fetched, res, rows.Err() } // LookupUserByID searches for user by ID @@ -535,34 +516,56 @@ func (s Store) internalUserEncoder(res *types.User) store.Payload { } } +// collectUserCursorValues collects values from the given resource that and sets them to the cursor +// to be used for pagination +// +// Values that are collected must come from sortable, unique or primary columns/fields +// At least one of the collected columns must be flagged as unique, otherwise fn appends primary keys at the end +// +// Known issue: +// when collecting cursor values for query that sorts by unique column with partial index (ie: unique handle on +// undeleted items) func (s Store) collectUserCursorValues(res *types.User, cc ...string) *filter.PagingCursor { var ( cursor = &filter.PagingCursor{} hasUnique bool + // All known primary key columns + + pkId bool + collect = func(cc ...string) { for _, c := range cc { switch c { case "id": cursor.Set(c, res.ID, false) + + pkId = true case "email": cursor.Set(c, res.Email, false) hasUnique = true + case "username": cursor.Set(c, res.Username, false) hasUnique = true + case "name": cursor.Set(c, res.Name, false) + case "handle": cursor.Set(c, res.Handle, false) hasUnique = true + case "created_at": cursor.Set(c, res.CreatedAt, false) + case "updated_at": cursor.Set(c, res.UpdatedAt, false) + case "suspended_at": cursor.Set(c, res.SuspendedAt, false) + case "deleted_at": cursor.Set(c, res.DeletedAt, false) @@ -572,10 +575,8 @@ func (s Store) collectUserCursorValues(res *types.User, cc ...string) *filter.Pa ) collect(cc...) - if !hasUnique { - collect( - "id", - ) + if !hasUnique || !(pkId && true) { + collect("id") } return cursor diff --git a/store/tests/compose_records_test.go b/store/tests/compose_records_test.go index fdfb77d1c..65d871158 100644 --- a/store/tests/compose_records_test.go +++ b/store/tests/compose_records_test.go @@ -6,6 +6,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/id" "github.com/cortezaproject/corteza-server/pkg/rh" "github.com/cortezaproject/corteza-server/store" + "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/require" "testing" "time" @@ -63,6 +64,32 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { return req, rr } + + stringifyValues = func(set types.RecordSet, fields ...string) string { + var out string + for r := range set { + if r > 0 { + out += ";" + } + + for f := range fields { + if f > 0 { + out += "," + } + + v := set[r].Values.Get(fields[f], 0) + if v != nil { + out += v.Value + } else { + out += "" + } + + } + + } + + return out + } ) t.Run("create", func(t *testing.T) { @@ -254,6 +281,92 @@ func testComposeRecords(t *testing.T, s store.ComposeRecords) { req.NoError(err) req.Len(set, 1) }) + + t.Run("sorted", func(t *testing.T) { + var ( + err error + set types.RecordSet + + req, _ = truncAndCreate(t, + makeNew(&types.RecordValue{Name: "str1", Value: "v1"}, &types.RecordValue{Name: "str3", Value: "three"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v2"}, &types.RecordValue{Name: "str3", Value: "three"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v3"}, &types.RecordValue{Name: "str3", Value: "three"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v4"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v5"}), + ) + + f = types.RecordFilter{ + ModuleID: mod.ID, + NamespaceID: mod.NamespaceID, + } + ) + + req.NoError(f.Sort.Set("str1")) + set, _, err = s.SearchComposeRecords(ctx, mod, f) + req.NoError(err) + req.Equal("v1,three;v2,three;v3,three;v4,;v5,", stringifyValues(set, "str1", "str3")) + + req.NoError(f.Sort.Set("str1 DESC")) + set, _, err = s.SearchComposeRecords(ctx, mod, f) + req.NoError(err) + req.Equal("v5,;v4,;v3,three;v2,three;v1,three", stringifyValues(set, "str1", "str3")) + + req.NoError(f.Sort.Set("str3")) + set, _, err = s.SearchComposeRecords(ctx, mod, f) + req.NoError(err) + req.Equal(",v4;,v5;three,v1;three,v2;three,v3", stringifyValues(set, "str3", "str1")) + + req.NoError(f.Sort.Set("str3 DESC")) + set, _, err = s.SearchComposeRecords(ctx, mod, f) + req.NoError(err) + req.Equal("three,v1;three,v2;three,v3;,v4;,v5", stringifyValues(set, "str3", "str1")) + + req.NoError(f.Sort.Set("str3 DESC, str1 DESC")) + set, _, err = s.SearchComposeRecords(ctx, mod, f) + req.NoError(err) + req.Equal("three,v3;three,v2;three,v1;,v5;,v4", stringifyValues(set, "str3", "str1")) + }) + + t.Run("paged", func(t *testing.T) { + var ( + err error + set types.RecordSet + + req, _ = truncAndCreate(t, + makeNew(&types.RecordValue{Name: "str1", Value: "v1"}, &types.RecordValue{Name: "str3", Value: "three"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v2"}, &types.RecordValue{Name: "str3", Value: "three"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v3"}, &types.RecordValue{Name: "str3", Value: "three"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v4"}), + makeNew(&types.RecordValue{Name: "str1", Value: "v5"}), + ) + + f = types.RecordFilter{ + ModuleID: mod.ID, + NamespaceID: mod.NamespaceID, + } + ) + + req.NoError(f.Sort.Set("str1")) + f.Limit = 3 + set, f, err = s.SearchComposeRecords(ctx, mod, f) + req.NoError(err) + req.NotNil(f.NextPage) + req.Nil(f.PrevPage) + req.Equal("v1,three;v2,three;v3,three", stringifyValues(set, "str1", "str3")) + + f.PageCursor = f.NextPage + set, f, err = s.SearchComposeRecords(ctx, mod, f) + req.Equal("v4,;v5,", stringifyValues(set, "str1", "str3")) + req.NoError(err) + + req.NoError(f.Sort.Set("str3 DESC")) + f.PageCursor = nil + f.Limit = 1 + set, f, err = s.SearchComposeRecords(ctx, mod, f) + spew.Dump(f) + req.Equal("three,v1", stringifyValues(set, "str3", "str1")) + req.NoError(err) + }) }) t.Run("report", func(t *testing.T) { diff --git a/store/tests/users_test.go b/store/tests/users_test.go index f070d6e8f..3fb7e63a1 100644 --- a/store/tests/users_test.go +++ b/store/tests/users_test.go @@ -53,6 +53,20 @@ func testUsers(t *testing.T, s store.Users) { return req, set } + stringifySetRange = func(set types.UserSet) string { + if len(set) == 0 { + return "" + } + + var out = set[0].Handle[7:] + + if len(set) > 1 { + out += ".." + set[len(set)-1].Handle[7:] + } + + return out + } + // in case we need some quick old-school debugging //dbg := func(uu ...*types.User) { // for i, u := range uu { @@ -216,8 +230,7 @@ func testUsers(t *testing.T, s store.Users) { req.Len(set, 3) req.NotNil(f.NextPage) req.Nil(f.PrevPage) - req.Equal("handle_01", set[0].Handle) - req.Equal("handle_03", set[2].Handle) + req.Equal("01..03", stringifySetRange(set)) // 2nd page f.Limit = 6 @@ -227,8 +240,7 @@ func testUsers(t *testing.T, s store.Users) { req.Len(set, 6) req.NotNil(f.NextPage) req.NotNil(f.PrevPage) - req.Equal("handle_04", set[0].Handle) - req.Equal("handle_09", set[5].Handle) + req.Equal("04..09", stringifySetRange(set)) // 3rd, last page (1 item left) f.Limit = 2 @@ -238,7 +250,7 @@ func testUsers(t *testing.T, s store.Users) { req.Len(set, 1) req.NotNil(f.NextPage) req.NotNil(f.PrevPage) - req.Equal("handle_10", set[0].Handle) + req.Equal("10", stringifySetRange(set)) // try and go pass the last page f.PageCursor = f.NextPage @@ -254,8 +266,7 @@ func testUsers(t *testing.T, s store.Users) { req.Len(set, 3) req.NotNil(f.NextPage) req.NotNil(f.PrevPage) - req.Equal("handle_07", set[0].Handle) - req.Equal("handle_09", set[2].Handle) + req.Equal("07..09", stringifySetRange(set)) // still in reverse, next 6 items f.Limit = 5 @@ -265,8 +276,7 @@ func testUsers(t *testing.T, s store.Users) { req.Len(set, 5) req.NotNil(f.NextPage) req.NotNil(f.PrevPage) - req.Equal("handle_02", set[0].Handle) - req.Equal("handle_06", set[4].Handle) + req.Equal("02..06", stringifySetRange(set)) // still in reverse, last 5 items (actually, we'll only get 1) f.Limit = 5 @@ -276,7 +286,7 @@ func testUsers(t *testing.T, s store.Users) { req.Len(set, 1) req.Nil(f.PrevPage) req.NotNil(f.NextPage) - req.Equal("handle_01", set[0].Handle) + req.Equal("01", stringifySetRange(set)) }) t.Run("with keyed paging and multi-key sorting", func(t *testing.T) {