Fix total and page navigation for records

Introduce helper func(IteratorPaging) to count total and create page navigation for records and also fixes timezone and milliseconds fir dateTime field value for query.

pkg/dal/iterator.go

@@ -3,6 +3,7 @@ package dal
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"github.com/cortezaproject/corteza-server/pkg/filter"
 	"io"
 )
@@ -63,3 +64,123 @@ func IteratorEncodeJSON(ctx context.Context, w io.Writer, iter Iterator, initTar
 
 	return
 }
+
+// IteratorPaging helper function for record paging cursor and total
+func IteratorPaging(ctx context.Context, iter Iterator, pp *filter.Paging, ss filter.Sorting, fn func(i Iterator) (ValueGetter, bool)) (err error) {
+	if pp == nil {
+		return
+	}
+
+	if pp.PageCursor != nil {
+		if pp.IncPageNavigation || pp.IncTotal {
+			return fmt.Errorf("not allowed to fetch page navigation or total item count with page cursor")
+		}
+	}
+
+	pp.Total = 0
+	pp.PrevPage = nil
+	pp.NextPage = nil
+	pp.PageNavigation = []*filter.Page{}
+
+	const howMuchMore = 1000
+
+	var (
+		counter   uint
+		total     uint
+		fetchMore bool
+
+		cur  *filter.PagingCursor
+		page = filter.Page{
+			Page:   1,
+			Count:  0,
+			Cursor: nil,
+		}
+	)
+
+	for iter.Next(ctx) {
+		if err = iter.Err(); err != nil {
+			return
+		}
+
+		r, ok := fn(iter)
+		if !ok {
+			continue
+		}
+
+		counter++
+		total++
+		page.Count++
+
+		if pp.PrevPage == nil {
+			pp.PrevPage, err = iter.BackCursor(r)
+			if err != nil {
+				return
+			}
+		}
+
+		// cursor for each page
+		cur, err = iter.ForwardCursor(r)
+		if err != nil {
+			return
+		}
+
+		if total%pp.Limit == 0 {
+			pp.PageNavigation = append(pp.PageNavigation, &filter.Page{
+				Page:   page.Page,
+				Count:  page.Count,
+				Cursor: page.Cursor,
+			})
+
+			// prep next page
+			page = filter.Page{
+				Page:   uint(len(pp.PageNavigation) + 1),
+				Count:  0,
+				Cursor: cur,
+			}
+
+			// fetch more records
+			fetchMore = true
+
+			if pp.NextPage == nil {
+				pp.NextPage = cur
+			}
+		}
+
+		if (len(pp.PageNavigation) == 1 && fetchMore) || counter == howMuchMore {
+			counter = 0
+			fetchMore = false
+
+			// request more items
+			if err = iter.More(howMuchMore, r); err != nil {
+				return
+			}
+		}
+	}
+
+	// push the last page to page navigation
+	if page.Count > 0 {
+		pp.PageNavigation = append(pp.PageNavigation, &filter.Page{
+			Page:   page.Page,
+			Count:  page.Count,
+			Cursor: page.Cursor,
+		})
+	}
+
+	if pp.PageCursor == nil {
+		pp.PrevPage = nil
+	}
+
+	if pp.NextPage != nil && len(pp.PageNavigation) == 1 {
+		pp.NextPage = nil
+	}
+
+	if pp.IncTotal {
+		pp.Total = total
+	}
+
+	if !pp.IncPageNavigation {
+		pp.PageNavigation = nil
+	}
+
+	return
+}