Enables paging & sorting for dal connection

Also, for the dal sensitivity level.

store/adapters/rdbms/rdbms.gen.go

@@ -9344,14 +9344,194 @@ func (s *Store) TruncateDalConnections(ctx context.Context) error {
 // This function is auto-generated
 func (s *Store) SearchDalConnections(ctx context.Context, f systemType.DalConnectionFilter) (set systemType.DalConnectionSet, _ systemType.DalConnectionFilter, err error) {
 
-	set, _, err = s.QueryDalConnections(ctx, f)
+	// Cleanup unwanted cursor values (only relevant is f.PageCursor, next&prev are reset and returned)
+	f.PrevPage, f.NextPage = nil, nil
+
+	if f.PageCursor != nil {
+		// Page cursor exists; we need to validate it against used sort
+		// To cover the case when paging cursor is set but sorting is empty, we collect the sorting instructions
+		// from the cursor.
+		// This (extracted sorting info) is then returned as part of response
+		if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
+			return
+		}
+	}
+
+	// Make sure results are always sorted at least by primary keys
+	if f.Sort.Get("id") == nil {
+		f.Sort = append(f.Sort, &filter.SortExpr{
+			Column:     "id",
+			Descending: f.Sort.LastDescending(),
+		})
+	}
+
+	// Cloned sorting instructions for the actual sorting
+	// Original are passed to the etchFullPageOfDalConnections fn used for cursor creation;
+	// direction information it MUST keep the initial
+	sort := f.Sort.Clone()
+
+	// When cursor for a previous page is used it's marked as reversed
+	// This tells us to flip the descending flag on all used sort keys
+	if f.PageCursor != nil && f.PageCursor.ROrder {
+		sort.Reverse()
+	}
+
+	set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfDalConnections(ctx, f, sort)
+
+	f.PageCursor = nil
 	if err != nil {
 		return nil, f, err
 	}
 
+	if f.IncTotal {
+		// Calc total from the number of items fetched
+		// even if we do build the page navigation
+		f.Total = uint(len(set))
+
+		if f.Limit > 0 && uint(len(set)) == f.Limit {
+			// there are fewer items fetched then requested limit
+			limit := f.Limit
+			f.Limit = 0
+			var navSet systemType.DalConnectionSet
+			if navSet, _, _, err = s.fetchFullPageOfDalConnections(ctx, f, sort); err != nil {
+				return
+			} else {
+				f.Total = uint(len(navSet))
+				f.Limit = limit
+			}
+		}
+	}
+
 	return set, f, nil
 }
 
+// fetchFullPageOfDalConnections collects all requested results.
+//
+// Function applies:
+//  - cursor conditions (where ...)
+//  - limit
+//
+// Main responsibility of this function is to perform additional sequential queries in case when not enough results
+// are collected due to failed check on a specific row (by check fn).
+//
+// Function then moves cursor to the last item fetched
+//
+// This function is auto-generated
+func (s *Store) fetchFullPageOfDalConnections(
+	ctx context.Context,
+	filter systemType.DalConnectionFilter,
+	sort filter.SortExprSet,
+) (set []*systemType.DalConnection, prev, next *filter.PagingCursor, err error) {
+	var (
+		aux []*systemType.DalConnection
+
+		// When cursor for a previous page is used it's marked as reversed
+		// This tells us to flip the descending flag on all used sort keys
+		reversedOrder = filter.PageCursor != nil && filter.PageCursor.ROrder
+
+		// Copy no. of required items to limit
+		// Limit will change when doing subsequent queries to fill
+		// the set with all required items
+		limit = filter.Limit
+
+		reqItems = filter.Limit
+
+		// cursor to prev. page is only calculated when cursor is used
+		hasPrev = filter.PageCursor != nil
+
+		// next cursor is calculated when there are more pages to come
+		hasNext bool
+
+		tryFilter systemType.DalConnectionFilter
+	)
+
+	set = make([]*systemType.DalConnection, 0, DefaultSliceCapacity)
+
+	for try := 0; try < MaxRefetches; try++ {
+		// Copy filter & apply custom sorting that might be affected by cursor
+		tryFilter = filter
+		tryFilter.Sort = sort
+
+		if limit > 0 {
+			// fetching + 1 to peak ahead if there are more items
+			// we can fetch (next-page cursor)
+			tryFilter.Limit = limit + 1
+		}
+
+		if aux, hasNext, err = s.QueryDalConnections(ctx, tryFilter); err != nil {
+			return nil, nil, nil, err
+		}
+
+		if len(aux) == 0 {
+			// nothing fetched
+			break
+		}
+
+		// append fetched items
+		set = append(set, aux...)
+
+		if reqItems == 0 || !hasNext {
+			// no max requested items specified, break out
+			break
+		}
+
+		collected := uint(len(set))
+
+		if reqItems > collected {
+			// not enough items fetched, try again with adjusted limit
+			limit = reqItems - collected
+
+			if limit < MinEnsureFetchLimit {
+				// In case limit is set very low and we've missed records in the first fetch,
+				// make sure next fetch limit is a bit higher
+				limit = MinEnsureFetchLimit
+			}
+
+			// Update cursor so that it points to the last item fetched
+			tryFilter.PageCursor = s.collectDalConnectionCursorValues(set[collected-1], filter.Sort...)
+
+			// Copy reverse flag from sorting
+			tryFilter.PageCursor.LThen = filter.Sort.Reversed()
+			continue
+		}
+
+		if reqItems < collected {
+			set = set[:reqItems]
+		}
+
+		break
+	}
+
+	collected := len(set)
+
+	if collected == 0 {
+		return nil, nil, nil, nil
+	}
+
+	if reversedOrder {
+		// Fetched set needs to be reversed because we've forced a descending order to get the previous page
+		for i, j := 0, collected-1; i < j; i, j = i+1, j-1 {
+			set[i], set[j] = set[j], set[i]
+		}
+
+		// when in reverse-order rules on what cursor to return change
+		hasPrev, hasNext = hasNext, hasPrev
+	}
+
+	if hasPrev {
+		prev = s.collectDalConnectionCursorValues(set[0], filter.Sort...)
+		prev.ROrder = true
+		prev.LThen = !filter.Sort.Reversed()
+	}
+
+	if hasNext {
+		next = s.collectDalConnectionCursorValues(set[collected-1], filter.Sort...)
+		next.LThen = filter.Sort.Reversed()
+	}
+
+	return set, prev, next, nil
+}
+
 // QueryDalConnections queries the database, converts and checks each row and returns collected set
 //
 // With generics, we can remove this per-resource-generated function
@@ -9371,6 +9551,8 @@ func (s *Store) QueryDalConnections(
 		rows        *sql.Rows
 		count       uint
 		expr, tExpr []goqu.Expression
+
+		sortExpr []exp.OrderedExpression
 	)
 
 	if s.Filters.DalConnection != nil {
@@ -9388,8 +9570,27 @@ func (s *Store) QueryDalConnections(
 
 	expr = append(expr, tExpr...)
 
+	// paging feature is enabled
+	if f.PageCursor != nil {
+		if tExpr, err = cursorWithSorting(f.PageCursor, s.sortableDalConnectionFields()); err != nil {
+			return
+		} else {
+			expr = append(expr, tExpr...)
+		}
+	}
+
 	query := dalConnectionSelectQuery(s.Dialect.GOQU()).Where(expr...)
 
+	// sorting feature is enabled
+	if sortExpr, err = order(f.Sort, s.sortableDalConnectionFields()); err != nil {
+		err = fmt.Errorf("could generate order expression for DalConnection: %w", err)
+		return
+	}
+
+	if len(sortExpr) > 0 {
+		query = query.Order(sortExpr...)
+	}
+
 	if f.Limit > 0 {
 		query = query.Limit(f.Limit)
 	}
@@ -9444,7 +9645,7 @@ func (s *Store) QueryDalConnections(
 		set = append(set, res)
 	}
 
-	return set, false, err
+	return set, f.Limit > 0 && count >= f.Limit, err
 
 }
 
@@ -9725,14 +9926,194 @@ func (s *Store) TruncateDalSensitivityLevels(ctx context.Context) error {
 // This function is auto-generated
 func (s *Store) SearchDalSensitivityLevels(ctx context.Context, f systemType.DalSensitivityLevelFilter) (set systemType.DalSensitivityLevelSet, _ systemType.DalSensitivityLevelFilter, err error) {
 
-	set, _, err = s.QueryDalSensitivityLevels(ctx, f)
+	// Cleanup unwanted cursor values (only relevant is f.PageCursor, next&prev are reset and returned)
+	f.PrevPage, f.NextPage = nil, nil
+
+	if f.PageCursor != nil {
+		// Page cursor exists; we need to validate it against used sort
+		// To cover the case when paging cursor is set but sorting is empty, we collect the sorting instructions
+		// from the cursor.
+		// This (extracted sorting info) is then returned as part of response
+		if f.Sort, err = f.PageCursor.Sort(f.Sort); err != nil {
+			return
+		}
+	}
+
+	// Make sure results are always sorted at least by primary keys
+	if f.Sort.Get("id") == nil {
+		f.Sort = append(f.Sort, &filter.SortExpr{
+			Column:     "id",
+			Descending: f.Sort.LastDescending(),
+		})
+	}
+
+	// Cloned sorting instructions for the actual sorting
+	// Original are passed to the etchFullPageOfDalSensitivityLevels fn used for cursor creation;
+	// direction information it MUST keep the initial
+	sort := f.Sort.Clone()
+
+	// When cursor for a previous page is used it's marked as reversed
+	// This tells us to flip the descending flag on all used sort keys
+	if f.PageCursor != nil && f.PageCursor.ROrder {
+		sort.Reverse()
+	}
+
+	set, f.PrevPage, f.NextPage, err = s.fetchFullPageOfDalSensitivityLevels(ctx, f, sort)
+
+	f.PageCursor = nil
 	if err != nil {
 		return nil, f, err
 	}
 
+	if f.IncTotal {
+		// Calc total from the number of items fetched
+		// even if we do build the page navigation
+		f.Total = uint(len(set))
+
+		if f.Limit > 0 && uint(len(set)) == f.Limit {
+			// there are fewer items fetched then requested limit
+			limit := f.Limit
+			f.Limit = 0
+			var navSet systemType.DalSensitivityLevelSet
+			if navSet, _, _, err = s.fetchFullPageOfDalSensitivityLevels(ctx, f, sort); err != nil {
+				return
+			} else {
+				f.Total = uint(len(navSet))
+				f.Limit = limit
+			}
+		}
+	}
+
 	return set, f, nil
 }
 
+// fetchFullPageOfDalSensitivityLevels collects all requested results.
+//
+// Function applies:
+//  - cursor conditions (where ...)
+//  - limit
+//
+// Main responsibility of this function is to perform additional sequential queries in case when not enough results
+// are collected due to failed check on a specific row (by check fn).
+//
+// Function then moves cursor to the last item fetched
+//
+// This function is auto-generated
+func (s *Store) fetchFullPageOfDalSensitivityLevels(
+	ctx context.Context,
+	filter systemType.DalSensitivityLevelFilter,
+	sort filter.SortExprSet,
+) (set []*systemType.DalSensitivityLevel, prev, next *filter.PagingCursor, err error) {
+	var (
+		aux []*systemType.DalSensitivityLevel
+
+		// When cursor for a previous page is used it's marked as reversed
+		// This tells us to flip the descending flag on all used sort keys
+		reversedOrder = filter.PageCursor != nil && filter.PageCursor.ROrder
+
+		// Copy no. of required items to limit
+		// Limit will change when doing subsequent queries to fill
+		// the set with all required items
+		limit = filter.Limit
+
+		reqItems = filter.Limit
+
+		// cursor to prev. page is only calculated when cursor is used
+		hasPrev = filter.PageCursor != nil
+
+		// next cursor is calculated when there are more pages to come
+		hasNext bool
+
+		tryFilter systemType.DalSensitivityLevelFilter
+	)
+
+	set = make([]*systemType.DalSensitivityLevel, 0, DefaultSliceCapacity)
+
+	for try := 0; try < MaxRefetches; try++ {
+		// Copy filter & apply custom sorting that might be affected by cursor
+		tryFilter = filter
+		tryFilter.Sort = sort
+
+		if limit > 0 {
+			// fetching + 1 to peak ahead if there are more items
+			// we can fetch (next-page cursor)
+			tryFilter.Limit = limit + 1
+		}
+
+		if aux, hasNext, err = s.QueryDalSensitivityLevels(ctx, tryFilter); err != nil {
+			return nil, nil, nil, err
+		}
+
+		if len(aux) == 0 {
+			// nothing fetched
+			break
+		}
+
+		// append fetched items
+		set = append(set, aux...)
+
+		if reqItems == 0 || !hasNext {
+			// no max requested items specified, break out
+			break
+		}
+
+		collected := uint(len(set))
+
+		if reqItems > collected {
+			// not enough items fetched, try again with adjusted limit
+			limit = reqItems - collected
+
+			if limit < MinEnsureFetchLimit {
+				// In case limit is set very low and we've missed records in the first fetch,
+				// make sure next fetch limit is a bit higher
+				limit = MinEnsureFetchLimit
+			}
+
+			// Update cursor so that it points to the last item fetched
+			tryFilter.PageCursor = s.collectDalSensitivityLevelCursorValues(set[collected-1], filter.Sort...)
+
+			// Copy reverse flag from sorting
+			tryFilter.PageCursor.LThen = filter.Sort.Reversed()
+			continue
+		}
+
+		if reqItems < collected {
+			set = set[:reqItems]
+		}
+
+		break
+	}
+
+	collected := len(set)
+
+	if collected == 0 {
+		return nil, nil, nil, nil
+	}
+
+	if reversedOrder {
+		// Fetched set needs to be reversed because we've forced a descending order to get the previous page
+		for i, j := 0, collected-1; i < j; i, j = i+1, j-1 {
+			set[i], set[j] = set[j], set[i]
+		}
+
+		// when in reverse-order rules on what cursor to return change
+		hasPrev, hasNext = hasNext, hasPrev
+	}
+
+	if hasPrev {
+		prev = s.collectDalSensitivityLevelCursorValues(set[0], filter.Sort...)
+		prev.ROrder = true
+		prev.LThen = !filter.Sort.Reversed()
+	}
+
+	if hasNext {
+		next = s.collectDalSensitivityLevelCursorValues(set[collected-1], filter.Sort...)
+		next.LThen = filter.Sort.Reversed()
+	}
+
+	return set, prev, next, nil
+}
+
 // QueryDalSensitivityLevels queries the database, converts and checks each row and returns collected set
 //
 // With generics, we can remove this per-resource-generated function
@@ -9752,6 +10133,8 @@ func (s *Store) QueryDalSensitivityLevels(
 		rows        *sql.Rows
 		count       uint
 		expr, tExpr []goqu.Expression
+
+		sortExpr []exp.OrderedExpression
 	)
 
 	if s.Filters.DalSensitivityLevel != nil {
@@ -9769,8 +10152,27 @@ func (s *Store) QueryDalSensitivityLevels(
 
 	expr = append(expr, tExpr...)
 
+	// paging feature is enabled
+	if f.PageCursor != nil {
+		if tExpr, err = cursorWithSorting(f.PageCursor, s.sortableDalSensitivityLevelFields()); err != nil {
+			return
+		} else {
+			expr = append(expr, tExpr...)
+		}
+	}
+
 	query := dalSensitivityLevelSelectQuery(s.Dialect.GOQU()).Where(expr...)
 
+	// sorting feature is enabled
+	if sortExpr, err = order(f.Sort, s.sortableDalSensitivityLevelFields()); err != nil {
+		err = fmt.Errorf("could generate order expression for DalSensitivityLevel: %w", err)
+		return
+	}
+
+	if len(sortExpr) > 0 {
+		query = query.Order(sortExpr...)
+	}
+
 	if f.Limit > 0 {
 		query = query.Limit(f.Limit)
 	}
@@ -9825,7 +10227,7 @@ func (s *Store) QueryDalSensitivityLevels(
 		set = append(set, res)
 	}
 
-	return set, false, err
+	return set, f.Limit > 0 && count >= f.Limit, err
 
 }
 

system/dal_connection.cue

@@ -51,8 +51,6 @@ dal_connection: {
 
 	features: {
 		labels: false
-		paging: false
-		sorting: false
 	}
 
 	rbac: {

system/dal_sensitivity_level.cue

@@ -45,8 +45,6 @@ dal_sensitivity_level: {
 
 	features: {
 		labels: false
-		paging: false
-		sorting: false
 	}
 
 	store: {

system/rest.yaml

@@ -773,6 +773,9 @@ endpoints:
       - name: deleted
         title: Exclude (0, default), include (1) or return only (2) deleted sensitivity levels
         type: uint
+      - type: bool
+        name: incTotal
+        title: Include total counter
 
   - name: create
     method: POST

system/rest/dal_connection.go

@@ -94,7 +94,7 @@ func (ctrl DalConnection) List(ctx context.Context, r *request.DalConnectionList
 
 	f.IncTotal = r.IncTotal
 
-	dalConnections, err = ctrl.collectConnections(ctx, f)
+	dalConnections, f, err = ctrl.collectConnections(ctx, f)
 	if err != nil {
 		return nil, err
 	}
@@ -208,14 +208,14 @@ func (ctrl DalConnection) federatedNodeToConnection(f *federationTypes.Node) *ty
 	}
 }
 
-func (ctrl DalConnection) collectConnections(ctx context.Context, f types.DalConnectionFilter) (out types.DalConnectionSet, err error) {
+func (ctrl DalConnection) collectConnections(ctx context.Context, filter types.DalConnectionFilter) (out types.DalConnectionSet, f types.DalConnectionFilter, err error) {
 	var (
 		dalConnections types.DalConnectionSet
 		federatedNodes federationTypes.NodeSet
 	)
 
-	if dalConnections, f, err = ctrl.svc.Search(ctx, f); err != nil {
-		return nil, err
+	if dalConnections, f, err = ctrl.svc.Search(ctx, filter); err != nil {
+		return nil, f, err
 	}
 
 	if !reflect2.IsNil(ctrl.federationSvc) {
@@ -223,7 +223,7 @@ func (ctrl DalConnection) collectConnections(ctx context.Context, f types.DalCon
 			// @todo IDs?
 			Deleted: f.Deleted,
 		}); err != nil {
-			return nil, err
+			return nil, f, err
 		}
 	}
 
@@ -233,8 +233,8 @@ func (ctrl DalConnection) collectConnections(ctx context.Context, f types.DalCon
 	// a unified output.
 	//
 	// Eventually federation nodes will become connections, so this is ok
-	for _, f := range federatedNodes {
-		out = append(out, ctrl.federatedNodeToConnection(f))
+	for _, nn := range federatedNodes {
+		out = append(out, ctrl.federatedNodeToConnection(nn))
 	}
 
 	out = ctrl.filterConnections(out, f)

system/rest/request/dalSensitivityLevel.go

@@ -44,6 +44,11 @@ type (
 		//
 		// Exclude (0, default), include (1) or return only (2) deleted sensitivity levels
 		Deleted uint
+
+		// IncTotal GET parameter
+		//
+		// Include total counter
+		IncTotal bool
 	}
 
 	DalSensitivityLevelCreate struct {
@@ -117,6 +122,7 @@ func (r DalSensitivityLevelList) Auditable() map[string]interface{} {
 	return map[string]interface{}{
 		"sensitivityLevelID": r.SensitivityLevelID,
 		"deleted":            r.Deleted,
+		"incTotal":           r.IncTotal,
 	}
 }
 
@@ -130,6 +136,11 @@ func (r DalSensitivityLevelList) GetDeleted() uint {
 	return r.Deleted
 }
 
+// Auditable returns all auditable/loggable parameters
+func (r DalSensitivityLevelList) GetIncTotal() bool {
+	return r.IncTotal
+}
+
 // Fill processes request and fills internal variables
 func (r *DalSensitivityLevelList) Fill(req *http.Request) (err error) {
 
@@ -154,6 +165,12 @@ func (r *DalSensitivityLevelList) Fill(req *http.Request) (err error) {
 				return err
 			}
 		}
+		if val, ok := tmp["incTotal"]; ok && len(val) > 0 {
+			r.IncTotal, err = payload.ParseBool(val[0]), nil
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	return err

system/rest/sensitivity_level.go

@@ -63,6 +63,8 @@ func (ctrl SensitivityLevel) List(ctx context.Context, r *request.DalSensitivity
 		f.Deleted = filter.StateExcluded
 	}
 
+	f.IncTotal = r.IncTotal
+
 	set, f, err = ctrl.svc.Search(ctx, f)
 	return ctrl.makeFilterPayload(ctx, set, f, err)
 }

system/types/dal_connection.go

@@ -111,6 +111,7 @@ type (
 
 		// Standard helpers for paging and sorting
 		filter.Paging
+		filter.Sorting
 	}
 )
 

system/types/dal_sensitivity_level.go

@@ -45,6 +45,7 @@ type (
 
 		// Standard helpers for paging and sorting
 		filter.Paging
+		filter.Sorting
 	}
 )