From 9afba4a6628e75adaac07769f14fde3f6f1e3530 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Toma=C5=BE=20Jerman?= <tomaz.jerman121@gmail.com>
Date: Thu, 18 Mar 2021 11:28:31 +0100
Subject: [PATCH] Add additional context to record import errors

---
 compose/rest/record.go    | 34 +++++++++++++++++++-----
 compose/service/record.go | 55 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 81 insertions(+), 8 deletions(-)

diff --git a/compose/rest/record.go b/compose/rest/record.go
index 6b383e393..5a3bbf1cd 100644
--- a/compose/rest/record.go
+++ b/compose/rest/record.go
@@ -346,17 +346,40 @@ func (ctrl *Record) ImportRun(ctx context.Context, r *request.RecordImportRun) (
 		cfg := &estore.EncoderConfig{
 			// For now the identifier is ignored, so this will never occur
 			OnExisting: resource.Skip,
-			Defer: func() {
+			DeferOk: func() {
 				ses.Progress.Completed++
 			},
 		}
-		if ses.OnError == service.IMPORT_ON_ERROR_SKIP {
-			cfg.DeferNok = func(err error) error {
-				ses.Progress.Failed++
-				ses.Progress.FailReason = err.Error()
+		cfg.DeferNok = func(err error) error {
+			ses.Progress.Failed++
 
+			if ses.Progress.FailLog == nil {
+				ses.Progress.FailLog = &service.FailLog{
+					Errors: make(service.ErrorIndex),
+				}
+			}
+
+			if rve, is := err.(*types.RecordValueErrorSet); is {
+				for _, ve := range rve.Set {
+					for k, v := range ve.Meta {
+						ses.Progress.FailLog.Errors.Add(fmt.Sprintf("%s %s %v", ve.Kind, k, v))
+					}
+				}
+			} else {
+				ses.Progress.FailLog.Errors.Add(err.Error())
+			}
+
+			if len(ses.Progress.FailLog.Records) < service.IMPORT_ERROR_MAX_INDEX_COUNT {
+				// +1 because we indexed them with 1 before
+				ses.Progress.FailLog.Records = append(ses.Progress.FailLog.Records, int(ses.Progress.Completed)+1)
+			} else {
+				ses.Progress.FailLog.RecordsTruncated = true
+			}
+
+			if ses.OnError == service.IMPORT_ON_ERROR_SKIP {
 				return nil
 			}
+			return err
 		}
 		se := estore.NewStoreEncoder(service.DefaultStore, cfg)
 		bld := envoy.NewBuilder(se)
@@ -371,7 +394,6 @@ func (ctrl *Record) ImportRun(ctx context.Context, r *request.RecordImportRun) (
 		ses.Progress.FinishedAt = &now
 		if err != nil {
 			ses.Progress.FailReason = err.Error()
-			ses.Progress.Failed++
 			return err
 		}
 
diff --git a/compose/service/record.go b/compose/service/record.go
index e190a1dbc..05618f727 100644
--- a/compose/service/record.go
+++ b/compose/service/record.go
@@ -2,8 +2,10 @@ package service
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"regexp"
+	"sort"
 	"strconv"
 	"time"
 
@@ -21,8 +23,9 @@ import (
 )
 
 const (
-	IMPORT_ON_ERROR_SKIP = "SKIP"
-	IMPORT_ON_ERROR_FAIL = "FAIL"
+	IMPORT_ON_ERROR_SKIP         = "SKIP"
+	IMPORT_ON_ERROR_FAIL         = "FAIL"
+	IMPORT_ERROR_MAX_INDEX_COUNT = 500000
 )
 
 type (
@@ -121,7 +124,20 @@ type (
 		Completed  uint64     `json:"completed"`
 		Failed     uint64     `json:"failed"`
 		FailReason string     `json:"failReason,omitempty"`
+
+		FailLog *FailLog `json:"failLog,omitempty"`
 	}
+
+	FailLog struct {
+		// Records holds an array of record indexes
+		Records          RecordIndex `json:"records"`
+		RecordsTruncated bool        `json:"recordsTruncated"`
+		// Errors specifies a map of occurred errors & the number of
+		Errors ErrorIndex `json:"errors"`
+	}
+
+	RecordIndex []int
+	ErrorIndex  map[string]int
 )
 
 func Record() RecordService {
@@ -1353,3 +1369,38 @@ func toLabeledRecords(set []*types.Record) []label.LabeledResource {
 
 	return ll
 }
+
+func (ei ErrorIndex) Add(err string) {
+	if _, has := ei[err]; has {
+		ei[err]++
+	} else {
+		ei[err] = 1
+	}
+}
+
+func (ri RecordIndex) MarshalJSON() ([]byte, error) {
+	sort.Ints(ri)
+
+	rr := make([][]int, 0, len(ri))
+	start := -1
+	crt := -1
+
+	for i := 0; i < len(ri); i++ {
+		if start == -1 {
+			start = ri[i]
+			crt = ri[i]
+			continue
+		}
+
+		// If the index increases for more then 1, the set is complete
+		if ri[i]-crt > 1 {
+			rr = append(rr, []int{start, crt})
+			start = ri[i]
+		}
+
+		crt = ri[i]
+	}
+
+	rr = append(rr, []int{start, crt})
+	return json.Marshal(rr)
+}