diff --git a/compose/types/attachment.go b/compose/types/attachment.go index a6724cb76..53900f53b 100644 --- a/compose/types/attachment.go +++ b/compose/types/attachment.go @@ -41,6 +41,12 @@ type ( // Standard paging fields & helpers rh.PageFilter + + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Attachment) (bool, error) } attachmentImageMeta struct { diff --git a/compose/types/chart.go b/compose/types/chart.go index 224b6eb87..618680dc7 100644 --- a/compose/types/chart.go +++ b/compose/types/chart.go @@ -54,6 +54,12 @@ type ( // Resource permission check filter IsReadable *permissions.ResourceFilter `json:"-"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Chart) (bool, error) + Deleted rh.FilterState `json:"deleted"` } ) diff --git a/compose/types/module.go b/compose/types/module.go index 0cffbc0a8..bcf2d480c 100644 --- a/compose/types/module.go +++ b/compose/types/module.go @@ -38,6 +38,12 @@ type ( // Resource permission check filter IsReadable *permissions.ResourceFilter `json:"-"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Module) (bool, error) + Deleted rh.FilterState `json:"deleted"` } ) diff --git a/compose/types/namespace.go b/compose/types/namespace.go index 784504adc..810ad4ffb 100644 --- a/compose/types/namespace.go +++ b/compose/types/namespace.go @@ -36,6 +36,12 @@ type ( // Resource permission check filter IsReadable *permissions.ResourceFilter `json:"-"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Namespace) (bool, error) + Deleted rh.FilterState `json:"deleted"` } diff --git a/compose/types/page.go b/compose/types/page.go index 9acbebdbc..e5d1afe32 100644 --- a/compose/types/page.go +++ b/compose/types/page.go @@ -67,6 +67,12 @@ type ( // Resource permission check filter IsReadable *permissions.ResourceFilter `json:"-"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Page) (bool, error) + Deleted rh.FilterState `json:"deleted"` } ) diff --git a/compose/types/record.go b/compose/types/record.go index ff3959ade..f455bc03e 100644 --- a/compose/types/record.go +++ b/compose/types/record.go @@ -57,6 +57,12 @@ type ( // Standard paging fields & helpers rh.PageFilter + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Record) (bool, error) + Deleted rh.FilterState `json:"deleted"` } ) diff --git a/pkg/codegen/assets/store_rdbms.gen.go.tpl b/pkg/codegen/assets/store_rdbms.gen.go.tpl index 63247da16..93101d0da 100644 --- a/pkg/codegen/assets/store_rdbms.gen.go.tpl +++ b/pkg/codegen/assets/store_rdbms.gen.go.tpl @@ -39,44 +39,139 @@ func (s Store) Search{{ pubIdent $.Types.Plural }}(ctx context.Context, f {{ $.T {{ if $.Search.DisablePaging }} scap := DefaultSliceCapacity {{ else }} - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } {{ end }} var ( set = make([]*{{ $.Types.GoType }}, 0, scap) - res *{{ $.Types.GoType }} - ) - return set, f, func() error { - {{- if not $.Search.DisablePaging }} - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - {{- end }} - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internal{{ pubIdent $.Types.Singular }}RowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } + {{- if $.Search.DisablePaging }} + fetch = func() error { + var ( + res *{{ $.Types.GoType }} + rows, err = s.Query(ctx, q) + ) + if err != nil { return err } - set = append(set, res) + for rows.Next() { + if res, err = s.internal{{ pubIdent $.Types.Singular }}RowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return err + } + + set = append(set, res) + } + + return rows.Close() + } + {{ else }} + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *{{ $.Types.GoType }} + chk bool + ) + + if limit > 0 { + q = q.Limit(uint64(limit)) + } + + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internal{{ pubIdent $.Types.Singular }}RowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + {{ end -}} + ) + + return set, f, fetch() } {{ end }} diff --git a/store/rdbms/actionlog.gen.go b/store/rdbms/actionlog.gen.go index 01de0b986..a0fe96342 100644 --- a/store/rdbms/actionlog.gen.go +++ b/store/rdbms/actionlog.gen.go @@ -31,30 +31,34 @@ func (s Store) SearchActionlogs(ctx context.Context, f actionlog.Filter) (action scap := DefaultSliceCapacity var ( - set = make([]*actionlog.Action, 0, scap) - res *actionlog.Action - ) - - return set, f, func() error { - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalActionlogRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } + set = make([]*actionlog.Action, 0, scap) + fetch = func() error { + var ( + res *actionlog.Action + rows, err = s.Query(ctx, q) + ) + if err != nil { return err } - set = append(set, res) - } + for rows.Next() { + if res, err = s.internalActionlogRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } - return rows.Close() - }() + return err + } + + set = append(set, res) + } + + return rows.Close() + } + ) + + return set, f, fetch() } // CreateActionlog creates one or more rows in actionlog table diff --git a/store/rdbms/applications.gen.go b/store/rdbms/applications.gen.go index c749a9994..0e3402ef9 100644 --- a/store/rdbms/applications.gen.go +++ b/store/rdbms/applications.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchApplications(ctx context.Context, f types.ApplicationFilter return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Application, 0, scap) - res *types.Application - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Application + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalApplicationRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalApplicationRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupApplicationByID searches for application by ID diff --git a/store/rdbms/attachments.gen.go b/store/rdbms/attachments.gen.go index 8f1f07c94..821026344 100644 --- a/store/rdbms/attachments.gen.go +++ b/store/rdbms/attachments.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchAttachments(ctx context.Context, f types.AttachmentFilter) return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Attachment, 0, scap) - res *types.Attachment - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Attachment + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalAttachmentRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalAttachmentRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupAttachmentByID searches for attachment by its ID diff --git a/store/rdbms/compose_charts.gen.go b/store/rdbms/compose_charts.gen.go index 3cc89166c..3122a1d39 100644 --- a/store/rdbms/compose_charts.gen.go +++ b/store/rdbms/compose_charts.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchComposeCharts(ctx context.Context, f types.ChartFilter) (ty return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Chart, 0, scap) - res *types.Chart - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Chart + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalComposeChartRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalComposeChartRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupComposeChartByID searches for compose chart by ID diff --git a/store/rdbms/compose_modules.gen.go b/store/rdbms/compose_modules.gen.go index 6e00a7ea2..c9212c307 100644 --- a/store/rdbms/compose_modules.gen.go +++ b/store/rdbms/compose_modules.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchComposeModules(ctx context.Context, f types.ModuleFilter) ( return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Module, 0, scap) - res *types.Module - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Module + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalComposeModuleRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalComposeModuleRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupComposeModuleByHandle searches for compose module by handle (case-insensitive) diff --git a/store/rdbms/compose_namespaces.gen.go b/store/rdbms/compose_namespaces.gen.go index 0f67a9099..bd5421f81 100644 --- a/store/rdbms/compose_namespaces.gen.go +++ b/store/rdbms/compose_namespaces.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchComposeNamespaces(ctx context.Context, f types.NamespaceFil return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Namespace, 0, scap) - res *types.Namespace - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Namespace + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalComposeNamespaceRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalComposeNamespaceRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupComposeNamespaceBySlug searches for namespace by slug (case-insensitive) diff --git a/store/rdbms/compose_pages.gen.go b/store/rdbms/compose_pages.gen.go index 49785d744..c533ba3f3 100644 --- a/store/rdbms/compose_pages.gen.go +++ b/store/rdbms/compose_pages.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchComposePages(ctx context.Context, f types.PageFilter) (type return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Page, 0, scap) - res *types.Page - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Page + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalComposePageRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalComposePageRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupComposePageByHandle searches for page chart by handle (case-insensitive) diff --git a/store/rdbms/credentials.gen.go b/store/rdbms/credentials.gen.go index daa4ebce7..96f256cd4 100644 --- a/store/rdbms/credentials.gen.go +++ b/store/rdbms/credentials.gen.go @@ -31,30 +31,34 @@ func (s Store) SearchCredentials(ctx context.Context, f types.CredentialsFilter) scap := DefaultSliceCapacity var ( - set = make([]*types.Credentials, 0, scap) - res *types.Credentials - ) - - return set, f, func() error { - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalCredentialsRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } + set = make([]*types.Credentials, 0, scap) + fetch = func() error { + var ( + res *types.Credentials + rows, err = s.Query(ctx, q) + ) + if err != nil { return err } - set = append(set, res) - } + for rows.Next() { + if res, err = s.internalCredentialsRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } - return rows.Close() - }() + return err + } + + set = append(set, res) + } + + return rows.Close() + } + ) + + return set, f, fetch() } // LookupCredentialsByID searches for credentials by ID diff --git a/store/rdbms/generic_selectors.go b/store/rdbms/generic_selectors.go index 30e2b4f50..e243175bc 100644 --- a/store/rdbms/generic_selectors.go +++ b/store/rdbms/generic_selectors.go @@ -27,28 +27,36 @@ func Count(ctx context.Context, db *sqlx.DB, q squirrel.SelectBuilder) (count ui return count, nil } -// FetchPaged fetches paged rows -func ApplyPaging(q squirrel.SelectBuilder, p rh.PageFilter) squirrel.SelectBuilder { - if p.Limit+p.Offset == 0 { +func calculatePaging(p rh.PageFilter) (o uint, l uint) { + o, l = p.Offset, p.Limit + + if o+l == 0 { // When both, offset & limit are 0, // calculate both values from page/perPage params if p.PerPage > 0 { - p.Limit = p.PerPage + l = p.PerPage } if p.Page < 1 { p.Page = 1 } - p.Offset = (p.Page - 1) * p.PerPage + o = (p.Page - 1) * p.PerPage } - if p.Limit > 0 { - q = q.Limit(uint64(p.Limit)) + return +} + +// FetchPaged fetches paged rows +func ApplyPaging(q squirrel.SelectBuilder, p rh.PageFilter) squirrel.SelectBuilder { + o, l := calculatePaging(p) + + if o > 0 { + q = q.Offset(uint64(o)) } - if p.Offset > 0 { - q = q.Offset(uint64(p.Offset)) + if l > 0 { + q = q.Limit(uint64(l)) } return q diff --git a/store/rdbms/rbac_rules.gen.go b/store/rdbms/rbac_rules.gen.go index b1131691e..d6e05ade1 100644 --- a/store/rdbms/rbac_rules.gen.go +++ b/store/rdbms/rbac_rules.gen.go @@ -28,30 +28,34 @@ func (s Store) SearchRbacRules(ctx context.Context, f permissions.RuleFilter) (p scap := DefaultSliceCapacity var ( - set = make([]*permissions.Rule, 0, scap) - res *permissions.Rule - ) - - return set, f, func() error { - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalRbacRuleRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } + set = make([]*permissions.Rule, 0, scap) + fetch = func() error { + var ( + res *permissions.Rule + rows, err = s.Query(ctx, q) + ) + if err != nil { return err } - set = append(set, res) - } + for rows.Next() { + if res, err = s.internalRbacRuleRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } - return rows.Close() - }() + return err + } + + set = append(set, res) + } + + return rows.Close() + } + ) + + return set, f, fetch() } // CreateRbacRule creates one or more rows in rbac_rules table diff --git a/store/rdbms/rdbms.go b/store/rdbms/rdbms.go index 14e8f4999..e8378b57c 100644 --- a/store/rdbms/rdbms.go +++ b/store/rdbms/rdbms.go @@ -78,6 +78,9 @@ const ( TxRetryHardLimit = 100 DefaultSliceCapacity = 1000 + + MinRefetchLimit = 10 + MaxRefetches = 100 ) var ( diff --git a/store/rdbms/reminders.gen.go b/store/rdbms/reminders.gen.go index d232dd879..275e615e1 100644 --- a/store/rdbms/reminders.gen.go +++ b/store/rdbms/reminders.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchReminders(ctx context.Context, f types.ReminderFilter) (typ return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Reminder, 0, scap) - res *types.Reminder - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Reminder + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalReminderRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalReminderRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupReminderByID searches for reminder by its ID diff --git a/store/rdbms/role_members.gen.go b/store/rdbms/role_members.gen.go index 96df08cc8..1bf23a5c7 100644 --- a/store/rdbms/role_members.gen.go +++ b/store/rdbms/role_members.gen.go @@ -28,30 +28,34 @@ func (s Store) SearchRoleMembers(ctx context.Context, f types.RoleMemberFilter) scap := DefaultSliceCapacity var ( - set = make([]*types.RoleMember, 0, scap) - res *types.RoleMember - ) - - return set, f, func() error { - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalRoleMemberRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } + set = make([]*types.RoleMember, 0, scap) + fetch = func() error { + var ( + res *types.RoleMember + rows, err = s.Query(ctx, q) + ) + if err != nil { return err } - set = append(set, res) - } + for rows.Next() { + if res, err = s.internalRoleMemberRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } - return rows.Close() - }() + return err + } + + set = append(set, res) + } + + return rows.Close() + } + ) + + return set, f, fetch() } // CreateRoleMember creates one or more rows in role_members table diff --git a/store/rdbms/roles.gen.go b/store/rdbms/roles.gen.go index 8d03092fc..b89a38310 100644 --- a/store/rdbms/roles.gen.go +++ b/store/rdbms/roles.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchRoles(ctx context.Context, f types.RoleFilter) (types.RoleS return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.Role, 0, scap) - res *types.Role - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.Role + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalRoleRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalRoleRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupRoleByID searches for role by ID diff --git a/store/rdbms/settings.gen.go b/store/rdbms/settings.gen.go index f76d3f6bf..7a5ea9e90 100644 --- a/store/rdbms/settings.gen.go +++ b/store/rdbms/settings.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchSettings(ctx context.Context, f types.SettingsFilter) (type return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.SettingValue, 0, scap) - res *types.SettingValue - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.SettingValue + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalSettingRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalSettingRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupSettingByNameOwnedBy searches for settings by name and owner diff --git a/store/rdbms/users.gen.go b/store/rdbms/users.gen.go index a2569636c..fb0ce331a 100644 --- a/store/rdbms/users.gen.go +++ b/store/rdbms/users.gen.go @@ -28,41 +28,110 @@ func (s Store) SearchUsers(ctx context.Context, f types.UserFilter) (types.UserS return nil, f, err } - q = ApplyPaging(q, f.PageFilter) - scap := f.PerPage if scap == 0 { scap = DefaultSliceCapacity } + if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { + return nil, f, err + } + var ( set = make([]*types.User, 0, scap) - res *types.User - ) + // @todo this offset needs to be removed and replaced with key-based-paging + fetchPage = func(offset, limit uint) (fetched, skipped uint, err error) { + var ( + res *types.User + chk bool + ) - return set, f, func() error { - if f.Count, err = Count(ctx, s.db, q); err != nil || f.Count == 0 { - return err - } - rows, err := s.Query(ctx, q) - if err != nil { - return err - } - - for rows.Next() { - if res, err = s.internalUserRowScanner(rows, rows.Err()); err != nil { - if cerr := rows.Close(); cerr != nil { - return fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) - } - - return err + if limit > 0 { + q = q.Limit(uint64(limit)) } - set = append(set, res) + if offset > 0 { + q = q.Offset(uint64(offset)) + } + + rows, err := s.Query(ctx, q) + if err != nil { + return + } + + for rows.Next() { + fetched++ + if res, err = s.internalUserRowScanner(rows, rows.Err()); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after scan error: %w", cerr, err) + } + + return + } + + // If check function is set, call it and act accordingly + if f.Check != nil { + if chk, err = f.Check(res); err != nil { + if cerr := rows.Close(); cerr != nil { + err = fmt.Errorf("could not close rows (%v) after check error: %w", cerr, err) + } + + return + } else if !chk { + // did not pass the check + // go with the next row + skipped++ + continue + } + } + + set = append(set, res) + + // make sure we do not fetch more than requested! + if f.Limit > 0 && uint(len(set)) >= f.Limit { + break + } + } + + err = rows.Close() + return } - return rows.Close() - }() + fetch = func() error { + var ( + fetched uint + + // starting offset & limit are from filter arg + // note that this will have to be improved with key-based pagination + offset, limit = calculatePaging(f.PageFilter) + ) + + for refetch := 0; refetch < MaxRefetches; refetch++ { + if fetched, _, err = fetchPage(offset, limit); err != nil { + return err + } + + // if limit is not set or we've already collected enough resources + // we can break the loop right away + if limit == 0 || fetched == 0 || uint(len(set)) >= f.Limit { + break + } + + // we've skipped fetched resources (due to check() fn) + // and we still have less results (in set) than required by limit + // inc offset by number of fetched items + offset += fetched + + if limit < MinRefetchLimit { + limit = MinRefetchLimit + } + + } + return nil + } + ) + + return set, f, fetch() } // LookupUserByID searches for user by ID diff --git a/store/rdbms/users.go b/store/rdbms/users.go index 864ff2976..9fc9ce02c 100644 --- a/store/rdbms/users.go +++ b/store/rdbms/users.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/Masterminds/squirrel" - "github.com/cortezaproject/corteza-server/pkg/permissions" "github.com/cortezaproject/corteza-server/pkg/rh" "github.com/cortezaproject/corteza-server/system/types" ) @@ -17,13 +16,13 @@ func (s Store) convertUserFilter(f types.UserFilter) (query squirrel.SelectBuild query = s.QueryUsers() // Returns user filter (flt) wrapped in IF() function with cnd as condition (when cnd != nil) - whereMasked := func(cnd *permissions.ResourceFilter, flt squirrel.Sqlizer) squirrel.Sqlizer { - if cnd != nil { - return rh.SquirrelFunction("IF", cnd, flt, squirrel.Expr("false")) - } else { - return flt - } - } + //whereMasked := func(cnd *permissions.ResourceFilter, flt squirrel.Sqlizer) squirrel.Sqlizer { + // if cnd != nil { + // return rh.SquirrelFunction("IF", cnd, flt, squirrel.Expr("false")) + // } else { + // return flt + // } + //} query = rh.FilterNullByState(query, "usr.deleted_at", f.Deleted) query = rh.FilterNullByState(query, "usr.suspended_at", f.Suspended) @@ -48,15 +47,19 @@ func (s Store) convertUserFilter(f types.UserFilter) (query squirrel.SelectBuild query = query.Where(squirrel.Or{ squirrel.Like{"usr.username": qs}, squirrel.Like{"usr.handle": qs}, - whereMasked(f.IsEmailUnmaskable, squirrel.Like{"usr.email": qs}), - whereMasked(f.IsNameUnmaskable, squirrel.Like{"usr.name": qs}), + //whereMasked(f.IsEmailUnmaskable, squirrel.Like{"usr.email": qs}), + //whereMasked(f.IsNameUnmaskable, squirrel.Like{"usr.name": qs}), }) } if f.Email != "" { - query = query.Where(whereMasked(f.IsEmailUnmaskable, squirrel.Eq{"usr.email": f.Email})) + query = query.Where(squirrel.Eq{"usr.email": f.Email}) } + //if f.Email != "" { + // query = query.Where(whereMasked(f.IsEmailUnmaskable, squirrel.Eq{"usr.email": f.Email})) + //} + if f.Username != "" { query = query.Where(squirrel.Eq{"usr.username": f.Username}) } @@ -69,9 +72,9 @@ func (s Store) convertUserFilter(f types.UserFilter) (query squirrel.SelectBuild query = query.Where(squirrel.Eq{"usr.kind": f.Kind}) } - if f.IsReadable != nil { - query = query.Where(f.IsReadable) - } + //if f.IsReadable != nil { + // query = query.Where(f.IsReadable) + //} var orderBy []string if orderBy, err = rh.ParseOrder(f.Sort, s.UserColumns()...); err != nil { diff --git a/store/tests/users_test.go b/store/tests/users_test.go index 6bd136a27..5eee91920 100644 --- a/store/tests/users_test.go +++ b/store/tests/users_test.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "github.com/cortezaproject/corteza-server/pkg/id" + "github.com/cortezaproject/corteza-server/pkg/rand" + "github.com/cortezaproject/corteza-server/pkg/rh" "github.com/cortezaproject/corteza-server/system/types" _ "github.com/joho/godotenv/autoload" "github.com/stretchr/testify/require" + "strings" "testing" "time" ) @@ -21,26 +24,52 @@ type ( func testUsers(t *testing.T, tmp interface{}) { var ( ctx = context.Background() - req = require.New(t) - - //err error - user *types.User s = tmp.(usersStoreAdt) + + makeNew = func(nn ...string) *types.User { + // minimum data set for new user + name := strings.Join(nn, "") + return &types.User{ + ID: id.Next(), + CreatedAt: time.Now(), + Email: "user-crud" + name + "@crust.test", + Username: "username_" + name, + Handle: "handle_" + name, + } + } + + truncAndCreate = func(t *testing.T) (*require.Assertions, *types.User) { + req := require.New(t) + req.NoError(s.TruncateUsers(ctx)) + user := makeNew() + req.NoError(s.CreateUser(ctx, user)) + return req, user + } + + truncAddFill = func(t *testing.T, l int) (*require.Assertions, types.UserSet) { + req := require.New(t) + req.NoError(s.TruncateUsers(ctx)) + + set := make([]*types.User, l) + + for i := 0; i < l; i++ { + set[i] = makeNew(string(rand.Bytes(10))) + } + + req.NoError(s.CreateUser(ctx, set...)) + return req, set + } ) t.Run("create", func(t *testing.T) { - user = &types.User{ - ID: 42, - CreatedAt: time.Now(), - Email: "user-crud@crust.test", - Username: "UserCRUD", - Handle: "usercrud", - } - req.NoError(s.CreateUser(ctx, user)) + req := require.New(t) + req.NoError(s.CreateUser(ctx, makeNew())) }) t.Run("lookup by ID", func(t *testing.T) { + req, user := truncAndCreate(t) + fetched, err := s.LookupUserByID(ctx, user.ID) req.NoError(err) req.Equal(user.Email, fetched.Email) @@ -54,13 +83,7 @@ func testUsers(t *testing.T, tmp interface{}) { }) t.Run("update", func(t *testing.T) { - user = &types.User{ - ID: 42, - CreatedAt: time.Now(), - Email: "user-crud+2@crust.test", - Username: "UserCRUD+2", - Handle: "usercrud+2", - } + req, user := truncAndCreate(t) req.NoError(s.UpdateUser(ctx, user)) }) @@ -94,80 +117,128 @@ func testUsers(t *testing.T, tmp interface{}) { //}) t.Run("lookup by email", func(t *testing.T) { + req, user := truncAndCreate(t) + fetched, err := s.LookupUserByEmail(ctx, user.Email) req.NoError(err) req.Equal(user.Email, fetched.Email) }) t.Run("lookup by handle", func(t *testing.T) { + req, user := truncAndCreate(t) + fetched, err := s.LookupUserByHandle(ctx, user.Handle) req.NoError(err) req.Equal(user.ID, fetched.ID) }) t.Run("lookup by nonexisting handle", func(t *testing.T) { + req, _ := truncAndCreate(t) + fetched, err := s.LookupUserByHandle(ctx, "no such handle") req.EqualError(err, "not found") req.Nil(fetched) }) t.Run("lookup by username", func(t *testing.T) { + req, user := truncAndCreate(t) + fetched, err := s.LookupUserByUsername(ctx, user.Username) req.NoError(err) req.Equal(user.ID, fetched.ID) }) - t.Run("search by ID", func(t *testing.T) { - set, f, err := s.SearchUsers(ctx, types.UserFilter{UserID: []uint64{user.ID}}) - req.NoError(err) - req.Equal([]uint64{user.ID}, f.UserID) - req.Len(set, 1) - req.Equal(uint(1), f.Count) - //req.Equal(set[0].ID, user.ID) - }) + t.Run("search", func(t *testing.T) { + t.Run("by ID", func(t *testing.T) { + req, prefill := truncAddFill(t, 5) - t.Run("search by email", func(t *testing.T) { - set, f, err := s.SearchUsers(ctx, types.UserFilter{Email: user.Email}) - req.NoError(err) - req.Len(set, 1) - req.Equal(uint(1), f.Count) - }) + set, f, err := s.SearchUsers(ctx, types.UserFilter{UserID: []uint64{prefill[0].ID}}) + req.NoError(err) + req.Equal([]uint64{prefill[0].ID}, f.UserID) + req.Len(set, 1) + req.Equal(uint(1), f.Count) + //req.Equal(set[0].ID, user.ID) + }) - t.Run("search by username", func(t *testing.T) { - set, f, err := s.SearchUsers(ctx, types.UserFilter{Username: user.Username}) - req.NoError(err) - req.Len(set, 1) - req.Equal(uint(1), f.Count) - }) + t.Run("by email", func(t *testing.T) { + req, prefill := truncAddFill(t, 5) - t.Run("search by query", func(t *testing.T) { - set, f, err := s.SearchUsers(ctx, types.UserFilter{Query: user.Handle}) - req.NoError(err) - req.Len(set, 1) - req.Equal(uint(1), f.Count) - }) + set, f, err := s.SearchUsers(ctx, types.UserFilter{Email: prefill[0].Email}) + req.NoError(err) + req.Len(set, 1) + req.Equal(uint(1), f.Count) + }) - t.Run("search by username", func(t *testing.T) { - set, f, err := s.SearchUsers(ctx, types.UserFilter{Username: "no such username"}) - req.NoError(err) - req.Len(set, 0) - req.Equal(uint(0), f.Count) - }) + t.Run("by username", func(t *testing.T) { + req, prefill := truncAddFill(t, 5) + set, f, err := s.SearchUsers(ctx, types.UserFilter{Username: prefill[0].Username}) + req.NoError(err) + req.Len(set, 1) + req.Equal(uint(1), f.Count) + }) - t.Run("search with masked details", func(t *testing.T) { - t.Skip("not implemented") - }) + t.Run("by query", func(t *testing.T) { + req, prefill := truncAddFill(t, 5) + set, f, err := s.SearchUsers(ctx, types.UserFilter{Query: prefill[0].Handle}) + req.NoError(err) + req.Len(set, 1) + req.Equal(uint(1), f.Count) + }) - t.Run("search by role", func(t *testing.T) { - t.Skip("not implemented") - }) + t.Run("by username", func(t *testing.T) { + req, _ := truncAddFill(t, 5) + set, f, err := s.SearchUsers(ctx, types.UserFilter{Username: "no such username"}) + req.NoError(err) + req.Len(set, 0) + req.Equal(uint(0), f.Count) + }) - t.Run("ordered search", func(t *testing.T) { - t.Skip("not implemented") + t.Run("with check", func(t *testing.T) { + req, prefill := truncAddFill(t, 5) + set, _, err := s.SearchUsers(ctx, types.UserFilter{ + Check: func(user *types.User) (bool, error) { + // simple check that matches with the first user from prefill + return user.ID == prefill[0].ID, nil + }, + }) + req.NoError(err) + req.Len(set, 1) + req.Equal(prefill[0].ID, set[0].ID) + }) + + t.Run("with check and paging", func(t *testing.T) { + req, prefill := truncAddFill(t, 5) + set, _, err := s.SearchUsers(ctx, types.UserFilter{ + // This will cause paging to run multiple queries + // until it collects all data + PageFilter: rh.PageFilter{Limit: 2}, + Check: func(user *types.User) (bool, error) { + // simple check that matches with the 4th user from prefill + return user.ID == prefill[4].ID, nil + }, + }) + req.NoError(err) + req.Len(set, 1) + req.Equal(prefill[4].ID, set[0].ID) + }) + + t.Run("with masked details", func(t *testing.T) { + t.Skip("not implemented") + }) + + t.Run("by role", func(t *testing.T) { + t.Skip("not implemented") + }) + + t.Run("search", func(t *testing.T) { + t.Skip("not implemented") + }) }) t.Run("count", func(t *testing.T) { var ( + req = require.New(t) + f = types.UserFilter{} c1, c2 uint err error diff --git a/system/service/access_control.go b/system/service/access_control.go index 3445e1cac..ac2c9aa5f 100644 --- a/system/service/access_control.go +++ b/system/service/access_control.go @@ -140,6 +140,10 @@ func (svc accessControl) FilterUsersWithUnmaskableName(ctx context.Context) *per return svc.filter(ctx, types.UserPermissionResource, "unmask.name", permissions.Deny) } +func (svc accessControl) CanReadUser(ctx context.Context, u *types.User) bool { + return svc.can(ctx, u.PermissionResource(), "read") +} + func (svc accessControl) CanUpdateUser(ctx context.Context, u *types.User) bool { return svc.can(ctx, u.PermissionResource(), "update") } diff --git a/system/service/user.go b/system/service/user.go index 503e5d3a3..8d20894d6 100644 --- a/system/service/user.go +++ b/system/service/user.go @@ -59,6 +59,7 @@ type ( userAccessController interface { CanAccess(context.Context) bool CanCreateUser(context.Context) bool + CanReadUser(context.Context, *types.User) bool CanUpdateUser(context.Context, *types.User) bool CanDeleteUser(context.Context, *types.User) bool CanSuspendUser(context.Context, *types.User) bool @@ -253,11 +254,23 @@ func (svc user) proc(u *types.User, err error) (*types.User, error) { return u, nil } +// Find interacts with backend storage and +// +// @todo rename to Search() for consistency func (svc user) Find(filter types.UserFilter) (uu types.UserSet, f types.UserFilter, err error) { var ( uaProps = &userActionProps{filter: &filter} ) + filter.Check = func(user *types.User) (bool, error) { + if !svc.ac.CanReadUser(svc.ctx, user) { + return false, nil + } + + svc.handlePrivateData(user) + return true, nil + } + err = func() error { if filter.Deleted > 0 { // If list with deleted users is requested @@ -270,13 +283,13 @@ func (svc user) Find(filter types.UserFilter) (uu types.UserSet, f types.UserFil } } - // Prepare filter for email unmasking check - filter.IsEmailUnmaskable = svc.ac.FilterUsersWithUnmaskableEmail(svc.ctx) - - // Prepare filter for name unmasking check - filter.IsNameUnmaskable = svc.ac.FilterUsersWithUnmaskableName(svc.ctx) - - filter.IsReadable = svc.ac.FilterReadableUsers(svc.ctx) + //// Prepare filter for email unmasking check + //filter.IsEmailUnmaskable = svc.ac.FilterUsersWithUnmaskableEmail(svc.ctx) + // + //// Prepare filter for name unmasking check + //filter.IsNameUnmaskable = svc.ac.FilterUsersWithUnmaskableName(svc.ctx) + // + //filter.IsReadable = svc.ac.FilterReadableUsers(svc.ctx) uu, f, err = svc.store.SearchUsers(svc.ctx, filter) if err != nil { diff --git a/system/types/applications.go b/system/types/applications.go index c2106c480..99015612f 100644 --- a/system/types/applications.go +++ b/system/types/applications.go @@ -45,8 +45,15 @@ type ( rh.PageFilter // Resource permission check filter + // deprecated IsReadable *permissions.ResourceFilter `json:"-"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Application) (bool, error) + Deleted rh.FilterState `json:"deleted"` } diff --git a/system/types/attachment.go b/system/types/attachment.go index 07c4a0cb2..410529c74 100644 --- a/system/types/attachment.go +++ b/system/types/attachment.go @@ -31,6 +31,12 @@ type ( Filter string `json:"filter"` Sort string `json:"sort"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Attachment) (bool, error) + // Standard paging fields & helpers rh.PageFilter } diff --git a/system/types/reminder.go b/system/types/reminder.go index 2cfe10e76..290ff216a 100644 --- a/system/types/reminder.go +++ b/system/types/reminder.go @@ -40,6 +40,12 @@ type ( Sort string `json:"sort"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Reminder) (bool, error) + // Standard paging fields & helpers rh.PageFilter } diff --git a/system/types/role.go b/system/types/role.go index db0e6e637..02255f435 100644 --- a/system/types/role.go +++ b/system/types/role.go @@ -37,7 +37,14 @@ type ( rh.PageFilter // Resource permission check filter + // deprecated IsReadable *permissions.ResourceFilter `json:"-"` + + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *Role) (bool, error) } RoleMetrics struct { diff --git a/system/types/settings.go b/system/types/settings.go index cb5a60ed3..2d4a70bd2 100644 --- a/system/types/settings.go +++ b/system/types/settings.go @@ -29,6 +29,12 @@ type ( OwnedBy uint64 `json:"ownedBy"` rh.PageFilter + + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *SettingValue) (bool, error) } SettingsKV map[string]types.JSONText diff --git a/system/types/user.go b/system/types/user.go index 1c72052f2..a2c7b05f1 100644 --- a/system/types/user.go +++ b/system/types/user.go @@ -54,14 +54,11 @@ type ( Sort string `json:"sort"` - // Can we use email for searching or is it supposed to be masked for the current user? - IsEmailUnmaskable *permissions.ResourceFilter `json:"-"` - - // Can we use name for searching or is it supposed to be masked for the current user? - IsNameUnmaskable *permissions.ResourceFilter `json:"-"` - - // Resource permission check filter - IsReadable *permissions.ResourceFilter `json:"-"` + // Check fn is called by store backend for each resource found function can + // modify the resource and return false if store should not return it + // + // Store then loads additional resources to satisfy the paging parameters + Check func(user *User) (bool, error) // Standard paging fields & helpers rh.PageFilter