Remove duplicate records with multi value fields

compose/types/record_value.go

@@ -4,11 +4,12 @@ import (
 	"database/sql/driver"
 	"encoding/json"
 	"fmt"
+	"strconv"
+	"time"
+
 	"github.com/cortezaproject/corteza-server/pkg/filter"
 	"github.com/pkg/errors"
 	"github.com/spf13/cast"
-	"strconv"
-	"time"
 )
 
 type (

pkg/codegen/assets/store_rdbms.gen.go.tpl

@@ -198,6 +198,11 @@ func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural  }} (
 		} else {
 			tryQuery = q
 		}
+		{{ if .RDBMS.CustomPreLoadProcessor }}
+		if tryQuery, err = s.{{ unexport $.Types.Singular }}PreLoadProcessor(tryQuery); err != nil {
+			return
+		}
+		{{- end}}
 
 		if limit > 0 {
 			// fetching + 1 so we know if there are more items

pkg/codegen/store.go

@@ -86,6 +86,7 @@ type (
 		CustomFilterConverter   bool `yaml:"customFilterConverter"`
 		CustomSortConverter     bool `yaml:"customSortConverter"`
 		CustomCursorCollector   bool `yaml:"customCursorCollector"`
+		CustomPreLoadProcessor  bool `yaml:"customPreLoadProcessor"`
 		CustomPostLoadProcessor bool `yaml:"customPostLoadProcessor"`
 		CustomEncoder           bool `yaml:"customEncoder"`
 

store/compose_records.yaml

@@ -48,6 +48,7 @@ rdbms:
   customFilterConverter: true
   customSortConverter: true
   customCursorCollector: true
+  customPreLoadProcessor: true
   customPostLoadProcessor: true
   mapFields:
     ModuleID:  { column: module_id }

store/rdbms/compose_records.gen.go

@@ -73,6 +73,10 @@ func (s Store) fetchFullPageOfComposeRecords(
 			tryQuery = q
 		}
 
+		if tryQuery, err = s.composeRecordPreLoadProcessor(tryQuery); err != nil {
+			return
+		}
+
 		if limit > 0 {
 			// fetching + 1 so we know if there are more items
 			// we can fetch (next-page cursor)

store/rdbms/compose_records.go

@@ -220,6 +220,10 @@ func (s Store) composeRecordsPageNavigation(
 				q = supPageQuery.Where(cursorCond(cursor))
 			}
 
+			if q, err = s.composeRecordPreLoadProcessor(q); err != nil {
+				return
+			}
+
 			rows, err = s.Query(ctx, q)
 			if err != nil {
 				return err
@@ -615,6 +619,16 @@ func (s Store) composeRecordPostLoadProcessor(ctx context.Context, m *types.Modu
 	return nil
 }
 
+func (s Store) composeRecordPreLoadProcessor(q squirrel.SelectBuilder) (squirrel.SelectBuilder, error) {
+	// When filtering over multi value record values we can get multiple rows of the same record
+	// causing it to show as a duplicate in the output.
+	// This partitioning removes any duplicate rows (rows where the index is not 1).
+	return squirrel.Select(s.composeRecordColumns()...).
+		PlaceholderFormat(s.config.PlaceholderFormat).
+		FromSelect(q.Column("row_number() over (partition by id) as pp_rn"), "base").
+		Where("pp_rn = 1"), nil
+}
+
 func (s Store) composeRecordsSorter(m *types.Module, q squirrel.SelectBuilder, sort filter.SortExprSet) (squirrel.SelectBuilder, error) {
 	var (
 		sortable = s.sortableComposeRecordColumns()

tests/compose/record_test.go

@@ -194,6 +194,34 @@ func TestRecordList(t *testing.T) {
 		End()
 }
 
+func TestRecordList_multiValueFilter(t *testing.T) {
+	h := newHelper(t)
+	h.clearRecords()
+
+	module := h.repoMakeRecordModuleWithFields("record testing module")
+
+	h.makeRecord(module, &types.RecordValue{Name: "options", Value: "a", Place: 0}, &types.RecordValue{Name: "options", Value: "b", Place: 1})
+	h.makeRecord(module, &types.RecordValue{Name: "options", Value: "a", Place: 0})
+	h.makeRecord(module, &types.RecordValue{Name: "options", Value: "c", Place: 0})
+	h.makeRecord(module, &types.RecordValue{Name: "options", Value: "a", Place: 0}, &types.RecordValue{Name: "options", Value: "b", Place: 1}, &types.RecordValue{Name: "options", Value: "c", Place: 2})
+
+	h.apiInit().
+		Get(fmt.Sprintf("/namespace/%d/module/%d/record/", module.NamespaceID, module.ID)).
+		Query("query", "options LIKE 'a' OR options LIKE 'b'").
+		Query("incTotal", "true").
+		Expect(t).
+		Status(http.StatusOK).
+		Assert(helpers.AssertNoErrors).
+		Assert(jsonpath.Equal(`$.response.filter.total`, float64(3))).
+		Assert(jsonpath.Equal(`$.response.set[0].values[0].value`, "a")).
+		Assert(jsonpath.Equal(`$.response.set[0].values[1].value`, "b")).
+		Assert(jsonpath.Equal(`$.response.set[1].values[0].value`, "a")).
+		Assert(jsonpath.Equal(`$.response.set[2].values[0].value`, "a")).
+		Assert(jsonpath.Equal(`$.response.set[2].values[1].value`, "b")).
+		Assert(jsonpath.Equal(`$.response.set[2].values[2].value`, "c")).
+		End()
+}
+
 func TestRecordListForbidenRecords(t *testing.T) {
 	h := newHelper(t)
 	h.clearRecords()