From 69b087549b51d43aaf38eaca51bc4ba2fad49cc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Toma=C5=BE=20Jerman?= <tomaz.jerman121@gmail.com>
Date: Thu, 16 Sep 2021 10:31:07 +0200
Subject: [PATCH] Remove duplicate records with multi value fields

---
 pkg/codegen/assets/store_rdbms.gen.go.tpl |  5 +++++
 pkg/codegen/store.go                      |  1 +
 store/compose_records.yaml                |  1 +
 store/rdbms/compose_records.gen.go        |  4 ++++
 store/rdbms/compose_records.go            | 14 ++++++++++++++
 5 files changed, 25 insertions(+)

diff --git a/pkg/codegen/assets/store_rdbms.gen.go.tpl b/pkg/codegen/assets/store_rdbms.gen.go.tpl
index 843a8d70c..d27e5daa9 100644
--- a/pkg/codegen/assets/store_rdbms.gen.go.tpl
+++ b/pkg/codegen/assets/store_rdbms.gen.go.tpl
@@ -198,6 +198,11 @@ func (s Store) {{ unexport "fetchFullPageOf" $.Types.Plural  }} (
 		} else {
 			tryQuery = q
 		}
+		{{ if .RDBMS.CustomPreLoadProcessor }}
+		if tryQuery, err = s.{{ unexport $.Types.Singular }}PreLoadProcessor(tryQuery); err != nil {
+			return
+		}
+		{{- end}}
 
 		if limit > 0 {
 			// fetching + 1 so we know if there are more items
diff --git a/pkg/codegen/store.go b/pkg/codegen/store.go
index 386c8f793..ee6e6f3fa 100644
--- a/pkg/codegen/store.go
+++ b/pkg/codegen/store.go
@@ -86,6 +86,7 @@ type (
 		CustomFilterConverter   bool `yaml:"customFilterConverter"`
 		CustomSortConverter     bool `yaml:"customSortConverter"`
 		CustomCursorCollector   bool `yaml:"customCursorCollector"`
+		CustomPreLoadProcessor  bool `yaml:"customPreLoadProcessor"`
 		CustomPostLoadProcessor bool `yaml:"customPostLoadProcessor"`
 		CustomEncoder           bool `yaml:"customEncoder"`
 
diff --git a/store/compose_records.yaml b/store/compose_records.yaml
index 7a31b73ba..b59f55217 100644
--- a/store/compose_records.yaml
+++ b/store/compose_records.yaml
@@ -56,6 +56,7 @@ rdbms:
   customFilterConverter: true
   customSortConverter: true
   customCursorCollector: true
+  customPreLoadProcessor: true
   customPostLoadProcessor: true
   mapFields:
     ModuleID:  { column: module_id }
diff --git a/store/rdbms/compose_records.gen.go b/store/rdbms/compose_records.gen.go
index 81811e923..2889a42a6 100644
--- a/store/rdbms/compose_records.gen.go
+++ b/store/rdbms/compose_records.gen.go
@@ -73,6 +73,10 @@ func (s Store) fetchFullPageOfComposeRecords(
 			tryQuery = q
 		}
 
+		if tryQuery, err = s.composeRecordPreLoadProcessor(tryQuery); err != nil {
+			return
+		}
+
 		if limit > 0 {
 			// fetching + 1 so we know if there are more items
 			// we can fetch (next-page cursor)
diff --git a/store/rdbms/compose_records.go b/store/rdbms/compose_records.go
index a0fd53e7a..67ddbb5c0 100644
--- a/store/rdbms/compose_records.go
+++ b/store/rdbms/compose_records.go
@@ -221,6 +221,10 @@ func (s Store) composeRecordsPageNavigation(
 				q = supPageQuery.Where(cursorCond(cursor))
 			}
 
+			if q, err = s.composeRecordPreLoadProcessor(q); err != nil {
+				return
+			}
+
 			rows, err = s.Query(ctx, q)
 			if err != nil {
 				return err
@@ -623,6 +627,16 @@ func (s Store) composeRecordPostLoadProcessor(ctx context.Context, m *types.Modu
 	return nil
 }
 
+func (s Store) composeRecordPreLoadProcessor(q squirrel.SelectBuilder) (squirrel.SelectBuilder, error) {
+	// When filtering over multi value record values we can get multiple rows of the same record
+	// causing it to show as a duplicate in the output.
+	// This partitioning removes any duplicate rows (rows where the index is not 1).
+	return squirrel.Select(s.composeRecordColumns()...).
+		PlaceholderFormat(s.config.PlaceholderFormat).
+		FromSelect(q.Column("row_number() over (partition by id) as pp_rn"), "base").
+		Where("pp_rn = 1"), nil
+}
+
 func (s Store) composeRecordsSorter(m *types.Module, q squirrel.SelectBuilder, sort filter.SortExprSet) (squirrel.SelectBuilder, error) {
 	var (
 		sortable = s.sortableComposeRecordColumns()