Extends DeDup capabilities

Support modifier for value search and allow ability to select matching criteria for multi value field

- Removed name from rule for now
- Value modifier to search with are ignore-case, case-sensitive, fuzzy-search, sounds-like
- Multi value matching criteria are one-of, equal
- Migrate RecordDeDup config for module, by adding upgrade fix for module.config.recordDeDup to migrate as per to the latest DeDupRule struct.

server/pkg/str/levenshtein.go

@@ -0,0 +1,52 @@
+package str
+
+// write Levenshtein Distance search algorithm for strings
+// https://en.wikipedia.org/wiki/Levenshtein_distance
+func ToLevenshteinDistance(a, b string) int {
+	var (
+		// length of a
+		la = len(a)
+		// length of b
+		lb = len(b)
+		// distance matrix
+		d = make([][]int, la+1)
+	)
+
+	// initialize distance matrix
+	for i := 0; i <= la; i++ {
+		d[i] = make([]int, lb+1)
+		d[i][0] = i
+	}
+
+	for j := 0; j <= lb; j++ {
+		d[0][j] = j
+	}
+
+	// calculate distance matrix
+	for i := 1; i <= la; i++ {
+		for j := 1; j <= lb; j++ {
+			if a[i-1] == b[j-1] {
+				d[i][j] = d[i-1][j-1]
+			} else {
+				// fix this min function
+				d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+1)
+			}
+		}
+	}
+
+	return d[la][lb]
+}
+
+func min(a, b, c int) int {
+	if a < b {
+		if a < c {
+			return a
+		}
+	}
+
+	if b < c {
+		return b
+	}
+
+	return c
+}

server/pkg/str/levenshtein_test.go

@@ -0,0 +1,39 @@
+package str
+
+import (
+	"testing"
+)
+
+func TestLevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		a    string
+		b    string
+		want int
+	}{
+		{"", "hello", 5},
+		{"hello", "", 5},
+		{"hello", "hello", 0},
+		{"ab", "aa", 1},
+		{"ab", "ba", 2},
+		{"ab", "aaa", 2},
+		{"bbb", "a", 3},
+		{"kitten", "sitting", 3},
+		{"distance", "difference", 5},
+		{"levenshtein", "frankenstein", 6},
+		{"resume and cafe", "resumes and cafes", 2},
+		{"a very long string that is meant to exceed", "another very long string that is meant to exceed", 6},
+		// Testing acutes and umlauts
+		{"resumé and café", "resumés and cafés", 2},
+		{"resume and cafe", "resumé and café", 4},
+		{"Hafþór Júlíus Björnsson", "Hafþor Julius Bjornsson", 8},
+		// Only 2 characters are less in the 2nd string
+		{"།་གམ་འས་པ་་མ།", "།་གམའས་པ་་མ", 6},
+	}
+	for _, tt := range tests {
+		t.Run(tt.a, func(t *testing.T) {
+			if got := ToLevenshteinDistance(tt.a, tt.b); got != tt.want {
+				t.Errorf("LevenshteinDistance() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

server/pkg/str/soundex.go

@@ -0,0 +1,104 @@
+package str
+
+import (
+	"strings"
+)
+
+// ToSoundex takes a word and returns the soundex code for it.
+// https://en.wikipedia.org/wiki/Soundex
+//
+// 1. Retain the first letter of the name and drop all other occurrences of a, e, i, o, u, y, h, w.
+// 2. Replace consonants with digits as follows (after the first letter):
+//		b, f, p, v → 1
+//		c, g, j, k, q, s, x, z → 2
+//		d, t → 3
+//		l → 4
+//		m, n → 5
+//		r → 6
+// 3. If two or more letters with the same number are adjacent in the original name (before step 1),
+//		only retain the first letter; also two letters with the same number separated
+//		by 'h' or 'w' are coded as a single number, whereas such letters separated by a vowel are coded twice.
+//		This rule also applies to the first letter.
+// 4. Iterate the previous step until you have one letter and three numbers.
+//		If you have too few letters in your word that you can't assign three numbers, append with zeros
+//		until there are three numbers. If you have more than 3 letters, just retain the first 3 numbers.
+func ToSoundex(s string) string {
+	var (
+		// soundex code
+		code string
+		// last code
+		lastCode string
+		// last rune
+		lastRune rune
+		// last rune is vowel
+		lastRuneIsVowel bool
+	)
+
+	// retain the first letter of the name and drop all other occurrences of a, e, i, o, u, y, h, w
+	for _, r := range s {
+		if r == 'a' || r == 'e' || r == 'i' || r == 'o' || r == 'u' || r == 'y' || r == 'h' || r == 'w' {
+			continue
+		}
+
+		code = string(r)
+		break
+	}
+
+	// replace consonants with digits as follows (after the first letter)
+	for _, r := range s {
+		if r == 'a' || r == 'e' || r == 'i' || r == 'o' || r == 'u' || r == 'y' || r == 'h' || r == 'w' {
+			lastRuneIsVowel = true
+			continue
+		}
+
+		if lastRuneIsVowel {
+			lastRuneIsVowel = false
+			lastCode = ""
+		}
+
+		switch r {
+		case 'b', 'f', 'p', 'v':
+			lastCode = "1"
+		case 'c', 'g', 'j', 'k', 'q', 's', 'x', 'z':
+			lastCode = "2"
+		case 'd', 't':
+			lastCode = "3"
+		case 'l':
+			lastCode = "4"
+		case 'm', 'n':
+			lastCode = "5"
+		case 'r':
+			lastCode = "6"
+		}
+
+		if lastCode != "" && lastCode != string(lastRune) {
+			code += lastCode
+		}
+
+		lastRune = r
+	}
+
+	// if two or more letters with the same number are adjacent in the original name (before step 1),
+	// only retain the first letter
+	// also two letters with the same number separated by 'h' or 'w' are coded as a single number,
+	// whereas such letters separated by a vowel are coded twice
+	// this rule also applies to the first letter
+	code = strings.ReplaceAll(code, "11", "1")
+	code = strings.ReplaceAll(code, "22", "2")
+	code = strings.ReplaceAll(code, "33", "3")
+	code = strings.ReplaceAll(code, "44", "4")
+	code = strings.ReplaceAll(code, "55", "5")
+	code = strings.ReplaceAll(code, "66", "6")
+
+	// iterate the previous step until you have one letter and three numbers
+	// if you have too few letters in your word that you can't assign three numbers,
+	// append with zeros until there are three numbers
+	// if you have more than 3 letters, just retain the first 3 numbers
+	if len(code) < 4 {
+		code += strings.Repeat("0", 4-len(code))
+	} else {
+		code = code[:4]
+	}
+
+	return code
+}

server/pkg/str/soundex_test.go

@@ -0,0 +1,64 @@
+package str
+
+import (
+	"testing"
+)
+
+func Test_soundex(t *testing.T) {
+	tests := []struct {
+		name string
+		want string
+	}{
+		{
+			"Robert",
+			"R163",
+		},
+		{
+			"Rupert",
+			"R163",
+		},
+		{
+			"Rubin",
+			"R150",
+		},
+		{
+			"Ashcraft",
+			"A261",
+		},
+		{
+			"Ashcroft",
+			"A261",
+		},
+		{
+			"Tymczak",
+			"T522",
+		},
+		{
+			"Pfister",
+			"P123",
+		},
+		{
+			"AH KEY",
+			"A000",
+		},
+		{
+			"The quick brown fox",
+			"T221",
+		},
+		{
+			"h3110 w021d",
+			"3000",
+		},
+		{
+			"1337",
+			"1000",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ToSoundex(tt.name); got != tt.want {
+				t.Errorf("soundex() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

server/pkg/str/str.go

@@ -0,0 +1,31 @@
+package str
+
+import (
+	"strings"
+)
+
+const (
+	// DefaultLevenshteinDistance is the default levenshtein distance
+	DefaultLevenshteinDistance = 3
+
+	CaseInSensitiveMatch = iota
+	CaseSensitiveMatch
+	LevenshteinDistance
+	Soundex
+)
+
+// Match will match string as per given algorithm
+func Match(str1, str2 string, algorithm int) bool {
+	switch algorithm {
+	case LevenshteinDistance:
+		return ToLevenshteinDistance(str1, str2) <= DefaultLevenshteinDistance
+	case Soundex:
+		return ToSoundex(str1) == ToSoundex(str2)
+	case CaseSensitiveMatch:
+		return strings.Compare(str1, str2) == 0
+	case CaseInSensitiveMatch:
+		return strings.EqualFold(str1, str2)
+	default:
+		return false
+	}
+}