Add --normalize option to normalize latin script characters

Close #790

CHANGELOG.md

@@ -6,6 +6,8 @@ CHANGELOG
 - Added `--height HEIGHT[%]` option
 - Preview window will truncate long lines by default. Line wrap can be enabled
   by `:wrap` flag in `--preview-window`.
+- Added `--normalize` option to normalize latin script letters before
+  matching. e.g. `sodanco` can match `Só Danço Samba`.
 
 0.15.9
 ------

man/man1/fzf.1

@@ -48,6 +48,10 @@ Case-insensitive match (default: smart-case match)
 .B "+i"
 Case-sensitive match
 .TP
+.B "--normalize"
+Normalize latin script letters before matching. This is not enabled by default
+to avoid performance overhead.
+.TP
 .BI "--algo=" TYPE
 Fuzzy matching algorithm (default: v2)
 

src/algo/algo.go

@@ -234,9 +234,36 @@ func bonusAt(input util.Chars, idx int) int16 {
 	return bonusFor(charClassOf(input.Get(idx-1)), charClassOf(input.Get(idx)))
 }
 
-type Algo func(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)
+func normalizeRune(r rune) rune {
+	if r < 0x00C0 || r > 0x2184 {
+		return r
+	}
 
-func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
+	n := normalized[r]
+	if n > 0 {
+		return n
+	}
+	return r
+}
+
+func normalizeRunes(runes []rune) []rune {
+	ret := make([]rune, len(runes))
+	copy(ret, runes)
+	for idx, r := range runes {
+		if r < 0x00C0 || r > 0x2184 {
+			continue
+		}
+		n := normalized[r]
+		if n > 0 {
+			ret[idx] = normalized[r]
+		}
+	}
+	return ret
+}
+
+type Algo func(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)
+
+func FuzzyMatchV2(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
 	// Assume that pattern is given in lowercase if case-insensitive.
 	// First check if there's a match and calculate bonus for each position.
 	// If the input string is too long, consider finding the matching chars in
@@ -247,13 +274,17 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
 	case 0:
 		return Result{0, 0, 0}, posArray(withPos, M)
 	case 1:
-		return ExactMatchNaive(caseSensitive, forward, input, pattern[0:1], withPos, slab)
+		return ExactMatchNaive(caseSensitive, normalize, forward, input, pattern[0:1], withPos, slab)
 	}
 
 	// Since O(nm) algorithm can be prohibitively expensive for large input,
 	// we fall back to the greedy algorithm.
 	if slab != nil && N*M > cap(slab.I16) {
-		return FuzzyMatchV1(caseSensitive, forward, input, pattern, withPos, slab)
+		return FuzzyMatchV1(caseSensitive, normalize, forward, input, pattern, withPos, slab)
+	}
+
+	if normalize {
+		pattern = normalizeRunes(pattern)
 	}
 
 	// Reuse pre-allocated integer slice to avoid unnecessary sweeping of garbages
@@ -285,6 +316,10 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
 			}
 		}
 
+		if normalize {
+			char = normalizeRune(char)
+		}
+
 		T[idx] = char
 		B[idx] = bonusFor(prevClass, class)
 		prevClass = class
@@ -432,7 +467,7 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
 }
 
 // Implement the same sorting criteria as V2
-func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {
+func calculateScore(caseSensitive bool, normalize bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {
 	pidx, score, inGap, consecutive, firstBonus := 0, 0, false, 0, int16(0)
 	pos := posArray(withPos, len(pattern))
 	prevClass := charNonWord
@@ -449,6 +484,10 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in
 				char = unicode.To(unicode.LowerCase, char)
 			}
 		}
+		// pattern is already normalized
+		if normalize {
+			char = normalizeRune(char)
+		}
 		if char == pattern[pidx] {
 			if withPos {
 				*pos = append(*pos, idx)
@@ -488,7 +527,7 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in
 }
 
 // FuzzyMatchV1 performs fuzzy-match
-func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
+func FuzzyMatchV1(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
 	if len(pattern) == 0 {
 		return Result{0, 0, 0}, nil
 	}
@@ -500,6 +539,10 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
 	lenRunes := text.Length()
 	lenPattern := len(pattern)
 
+	if normalize {
+		pattern = normalizeRunes(pattern)
+	}
+
 	for index := 0; index < lenRunes; index++ {
 		char := text.Get(indexAt(index, lenRunes, forward))
 		// This is considerably faster than blindly applying strings.ToLower to the
@@ -514,6 +557,9 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
 				char = unicode.To(unicode.LowerCase, char)
 			}
 		}
+		if normalize {
+			char = normalizeRune(char)
+		}
 		pchar := pattern[indexAt(pidx, lenPattern, forward)]
 		if char == pchar {
 			if sidx < 0 {
@@ -553,7 +599,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
 			sidx, eidx = lenRunes-eidx, lenRunes-sidx
 		}
 
-		score, pos := calculateScore(caseSensitive, text, pattern, sidx, eidx, withPos)
+		score, pos := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, withPos)
 		return Result{sidx, eidx, score}, pos
 	}
 	return Result{-1, -1, 0}, nil
@@ -568,7 +614,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
 // bonus point, instead of stopping immediately after finding the first match.
 // The solution is much cheaper since there is only one possible alignment of
 // the pattern.
-func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
+func ExactMatchNaive(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
 	if len(pattern) == 0 {
 		return Result{0, 0, 0}, nil
 	}
@@ -580,6 +626,10 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
 		return Result{-1, -1, 0}, nil
 	}
 
+	if normalize {
+		pattern = normalizeRunes(pattern)
+	}
+
 	// For simplicity, only look at the bonus at the first character position
 	pidx := 0
 	bestPos, bonus, bestBonus := -1, int16(0), int16(-1)
@@ -593,6 +643,9 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
 				char = unicode.To(unicode.LowerCase, char)
 			}
 		}
+		if normalize {
+			char = normalizeRune(char)
+		}
 		pidx_ := indexAt(pidx, lenPattern, forward)
 		pchar := pattern[pidx_]
 		if pchar == char {
@@ -624,14 +677,14 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
 			sidx = lenRunes - (bestPos + 1)
 			eidx = lenRunes - (bestPos - lenPattern + 1)
 		}
-		score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false)
+		score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)
 		return Result{sidx, eidx, score}, nil
 	}
 	return Result{-1, -1, 0}, nil
 }
 
 // PrefixMatch performs prefix-match
-func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
+func PrefixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
 	if len(pattern) == 0 {
 		return Result{0, 0, 0}, nil
 	}
@@ -640,22 +693,29 @@ func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
 		return Result{-1, -1, 0}, nil
 	}
 
+	if normalize {
+		pattern = normalizeRunes(pattern)
+	}
+
 	for index, r := range pattern {
 		char := text.Get(index)
 		if !caseSensitive {
 			char = unicode.ToLower(char)
 		}
+		if normalize {
+			char = normalizeRune(char)
+		}
 		if char != r {
 			return Result{-1, -1, 0}, nil
 		}
 	}
 	lenPattern := len(pattern)
-	score, _ := calculateScore(caseSensitive, text, pattern, 0, lenPattern, false)
+	score, _ := calculateScore(caseSensitive, normalize, text, pattern, 0, lenPattern, false)
 	return Result{0, lenPattern, score}, nil
 }
 
 // SuffixMatch performs suffix-match
-func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
+func SuffixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
 	lenRunes := text.Length()
 	trimmedLen := lenRunes - text.TrailingWhitespaces()
 	if len(pattern) == 0 {
@@ -666,11 +726,18 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
 		return Result{-1, -1, 0}, nil
 	}
 
+	if normalize {
+		pattern = normalizeRunes(pattern)
+	}
+
 	for index, r := range pattern {
 		char := text.Get(index + diff)
 		if !caseSensitive {
 			char = unicode.ToLower(char)
 		}
+		if normalize {
+			char = normalizeRune(char)
+		}
 		if char != r {
 			return Result{-1, -1, 0}, nil
 		}
@@ -678,21 +745,37 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
 	lenPattern := len(pattern)
 	sidx := trimmedLen - lenPattern
 	eidx := trimmedLen
-	score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false)
+	score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)
 	return Result{sidx, eidx, score}, nil
 }
 
 // EqualMatch performs equal-match
-func EqualMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
+func EqualMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
 	lenPattern := len(pattern)
 	if text.Length() != lenPattern {
 		return Result{-1, -1, 0}, nil
 	}
+	match := true
+	if normalize {
+		runes := text.ToRunes()
+		for idx, pchar := range pattern {
+			char := runes[idx]
+			if !caseSensitive {
+				char = unicode.To(unicode.LowerCase, char)
+			}
+			if normalizeRune(pchar) != normalizeRune(char) {
+				match = false
+				break
+			}
+		}
+	} else {
 		runesStr := text.ToString()
 		if !caseSensitive {
 			runesStr = strings.ToLower(runesStr)
 		}
-	if runesStr == string(pattern) {
+		match = runesStr == string(pattern)
+	}
+	if match {
 		return Result{0, lenPattern, (scoreMatch+bonusBoundary)*lenPattern +
 			(bonusFirstCharMultiplier-1)*bonusBoundary}, nil
 	}

src/algo/algo_test.go

@@ -10,10 +10,14 @@ import (
 )
 
 func assertMatch(t *testing.T, fun Algo, caseSensitive, forward bool, input, pattern string, sidx int, eidx int, score int) {
+	assertMatch2(t, fun, caseSensitive, false, forward, input, pattern, sidx, eidx, score)
+}
+
+func assertMatch2(t *testing.T, fun Algo, caseSensitive, normalize, forward bool, input, pattern string, sidx int, eidx int, score int) {
 	if !caseSensitive {
 		pattern = strings.ToLower(pattern)
 	}
-	res, pos := fun(caseSensitive, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil)
+	res, pos := fun(caseSensitive, normalize, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil)
 	var start, end int
 	if pos == nil || len(*pos) == 0 {
 		start = res.Start
@@ -156,6 +160,21 @@ func TestEmptyPattern(t *testing.T) {
 	}
 }
 
+func TestNormalize(t *testing.T) {
+	caseSensitive := false
+	normalize := true
+	forward := true
+	test := func(input, pattern string, sidx, eidx, score int, funs ...Algo) {
+		for _, fun := range funs {
+			assertMatch2(t, fun, caseSensitive, normalize, forward,
+				input, pattern, sidx, eidx, score)
+		}
+	}
+	test("Só Danço Samba", "So", 0, 2, 56, FuzzyMatchV1, FuzzyMatchV2, PrefixMatch, ExactMatchNaive)
+	test("Só Danço Samba", "sodc", 0, 7, 89, FuzzyMatchV1, FuzzyMatchV2)
+	test("Danço", "danco", 0, 5, 128, FuzzyMatchV1, FuzzyMatchV2, PrefixMatch, SuffixMatch, ExactMatchNaive, EqualMatch)
+}
+
 func TestLongString(t *testing.T) {
 	bytes := make([]byte, math.MaxUint16*2)
 	for i := range bytes {

src/algo/normalize.go

@@ -0,0 +1,408 @@
+// Normalization of latin script letters
+// Reference: http://www.unicode.org/Public/UCD/latest/ucd/Index.txt
+
+package algo
+
+var normalized map[rune]rune = map[rune]rune{
+	0x00E1: 'a', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0103: 'a', //  WITH BREVE, LATIN SMALL LETTER
+	0x01CE: 'a', //  WITH CARON, LATIN SMALL LETTER
+	0x00E2: 'a', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x00E4: 'a', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x0227: 'a', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1EA1: 'a', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0201: 'a', //  WITH DOUBLE GRAVE, LATIN SMALL LETTER
+	0x00E0: 'a', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1EA3: 'a', //  WITH HOOK ABOVE, LATIN SMALL LETTER
+	0x0203: 'a', //  WITH INVERTED BREVE, LATIN SMALL LETTER
+	0x0101: 'a', //  WITH MACRON, LATIN SMALL LETTER
+	0x0105: 'a', //  WITH OGONEK, LATIN SMALL LETTER
+	0x1E9A: 'a', //  WITH RIGHT HALF RING, LATIN SMALL LETTER
+	0x00E5: 'a', //  WITH RING ABOVE, LATIN SMALL LETTER
+	0x1E01: 'a', //  WITH RING BELOW, LATIN SMALL LETTER
+	0x00E3: 'a', //  WITH TILDE, LATIN SMALL LETTER
+	0x0363: 'a', // , COMBINING LATIN SMALL LETTER
+	0x0250: 'a', // , LATIN SMALL LETTER TURNED
+	0x1E03: 'b', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E05: 'b', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0253: 'b', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E07: 'b', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x0180: 'b', //  WITH STROKE, LATIN SMALL LETTER
+	0x0183: 'b', //  WITH TOPBAR, LATIN SMALL LETTER
+	0x0107: 'c', //  WITH ACUTE, LATIN SMALL LETTER
+	0x010D: 'c', //  WITH CARON, LATIN SMALL LETTER
+	0x00E7: 'c', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x0109: 'c', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x0255: 'c', //  WITH CURL, LATIN SMALL LETTER
+	0x010B: 'c', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x0188: 'c', //  WITH HOOK, LATIN SMALL LETTER
+	0x023C: 'c', //  WITH STROKE, LATIN SMALL LETTER
+	0x0368: 'c', // , COMBINING LATIN SMALL LETTER
+	0x0297: 'c', // , LATIN LETTER STRETCHED
+	0x2184: 'c', // , LATIN SMALL LETTER REVERSED
+	0x010F: 'd', //  WITH CARON, LATIN SMALL LETTER
+	0x1E11: 'd', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E13: 'd', //  WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
+	0x0221: 'd', //  WITH CURL, LATIN SMALL LETTER
+	0x1E0B: 'd', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E0D: 'd', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0257: 'd', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E0F: 'd', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x0111: 'd', //  WITH STROKE, LATIN SMALL LETTER
+	0x0256: 'd', //  WITH TAIL, LATIN SMALL LETTER
+	0x018C: 'd', //  WITH TOPBAR, LATIN SMALL LETTER
+	0x0369: 'd', // , COMBINING LATIN SMALL LETTER
+	0x00E9: 'e', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0115: 'e', //  WITH BREVE, LATIN SMALL LETTER
+	0x011B: 'e', //  WITH CARON, LATIN SMALL LETTER
+	0x0229: 'e', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E19: 'e', //  WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
+	0x00EA: 'e', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x00EB: 'e', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x0117: 'e', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1EB9: 'e', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0205: 'e', //  WITH DOUBLE GRAVE, LATIN SMALL LETTER
+	0x00E8: 'e', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1EBB: 'e', //  WITH HOOK ABOVE, LATIN SMALL LETTER
+	0x025D: 'e', //  WITH HOOK, LATIN SMALL LETTER REVERSED OPEN
+	0x0207: 'e', //  WITH INVERTED BREVE, LATIN SMALL LETTER
+	0x0113: 'e', //  WITH MACRON, LATIN SMALL LETTER
+	0x0119: 'e', //  WITH OGONEK, LATIN SMALL LETTER
+	0x0247: 'e', //  WITH STROKE, LATIN SMALL LETTER
+	0x1E1B: 'e', //  WITH TILDE BELOW, LATIN SMALL LETTER
+	0x1EBD: 'e', //  WITH TILDE, LATIN SMALL LETTER
+	0x0364: 'e', // , COMBINING LATIN SMALL LETTER
+	0x029A: 'e', // , LATIN SMALL LETTER CLOSED OPEN
+	0x025E: 'e', // , LATIN SMALL LETTER CLOSED REVERSED OPEN
+	0x025B: 'e', // , LATIN SMALL LETTER OPEN
+	0x0258: 'e', // , LATIN SMALL LETTER REVERSED
+	0x025C: 'e', // , LATIN SMALL LETTER REVERSED OPEN
+	0x01DD: 'e', // , LATIN SMALL LETTER TURNED
+	0x1D08: 'e', // , LATIN SMALL LETTER TURNED OPEN
+	0x1E1F: 'f', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x0192: 'f', //  WITH HOOK, LATIN SMALL LETTER
+	0x01F5: 'g', //  WITH ACUTE, LATIN SMALL LETTER
+	0x011F: 'g', //  WITH BREVE, LATIN SMALL LETTER
+	0x01E7: 'g', //  WITH CARON, LATIN SMALL LETTER
+	0x0123: 'g', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x011D: 'g', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x0121: 'g', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x0260: 'g', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E21: 'g', //  WITH MACRON, LATIN SMALL LETTER
+	0x01E5: 'g', //  WITH STROKE, LATIN SMALL LETTER
+	0x0261: 'g', // , LATIN SMALL LETTER SCRIPT
+	0x1E2B: 'h', //  WITH BREVE BELOW, LATIN SMALL LETTER
+	0x021F: 'h', //  WITH CARON, LATIN SMALL LETTER
+	0x1E29: 'h', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x0125: 'h', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x1E27: 'h', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1E23: 'h', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E25: 'h', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x02AE: 'h', //  WITH FISHHOOK, LATIN SMALL LETTER TURNED
+	0x0266: 'h', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E96: 'h', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x0127: 'h', //  WITH STROKE, LATIN SMALL LETTER
+	0x036A: 'h', // , COMBINING LATIN SMALL LETTER
+	0x0265: 'h', // , LATIN SMALL LETTER TURNED
+	0x2095: 'h', // , LATIN SUBSCRIPT SMALL LETTER
+	0x00ED: 'i', //  WITH ACUTE, LATIN SMALL LETTER
+	0x012D: 'i', //  WITH BREVE, LATIN SMALL LETTER
+	0x01D0: 'i', //  WITH CARON, LATIN SMALL LETTER
+	0x00EE: 'i', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x00EF: 'i', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1ECB: 'i', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0209: 'i', //  WITH DOUBLE GRAVE, LATIN SMALL LETTER
+	0x00EC: 'i', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1EC9: 'i', //  WITH HOOK ABOVE, LATIN SMALL LETTER
+	0x020B: 'i', //  WITH INVERTED BREVE, LATIN SMALL LETTER
+	0x012B: 'i', //  WITH MACRON, LATIN SMALL LETTER
+	0x012F: 'i', //  WITH OGONEK, LATIN SMALL LETTER
+	0x0268: 'i', //  WITH STROKE, LATIN SMALL LETTER
+	0x1E2D: 'i', //  WITH TILDE BELOW, LATIN SMALL LETTER
+	0x0129: 'i', //  WITH TILDE, LATIN SMALL LETTER
+	0x0365: 'i', // , COMBINING LATIN SMALL LETTER
+	0x0131: 'i', // , LATIN SMALL LETTER DOTLESS
+	0x1D09: 'i', // , LATIN SMALL LETTER TURNED
+	0x1D62: 'i', // , LATIN SUBSCRIPT SMALL LETTER
+	0x2071: 'i', // , SUPERSCRIPT LATIN SMALL LETTER
+	0x01F0: 'j', //  WITH CARON, LATIN SMALL LETTER
+	0x0135: 'j', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x029D: 'j', //  WITH CROSSED-TAIL, LATIN SMALL LETTER
+	0x0249: 'j', //  WITH STROKE, LATIN SMALL LETTER
+	0x025F: 'j', //  WITH STROKE, LATIN SMALL LETTER DOTLESS
+	0x0237: 'j', // , LATIN SMALL LETTER DOTLESS
+	0x1E31: 'k', //  WITH ACUTE, LATIN SMALL LETTER
+	0x01E9: 'k', //  WITH CARON, LATIN SMALL LETTER
+	0x0137: 'k', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E33: 'k', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0199: 'k', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E35: 'k', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x029E: 'k', // , LATIN SMALL LETTER TURNED
+	0x2096: 'k', // , LATIN SUBSCRIPT SMALL LETTER
+	0x013A: 'l', //  WITH ACUTE, LATIN SMALL LETTER
+	0x019A: 'l', //  WITH BAR, LATIN SMALL LETTER
+	0x026C: 'l', //  WITH BELT, LATIN SMALL LETTER
+	0x013E: 'l', //  WITH CARON, LATIN SMALL LETTER
+	0x013C: 'l', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E3D: 'l', //  WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
+	0x0234: 'l', //  WITH CURL, LATIN SMALL LETTER
+	0x1E37: 'l', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x1E3B: 'l', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x0140: 'l', //  WITH MIDDLE DOT, LATIN SMALL LETTER
+	0x026B: 'l', //  WITH MIDDLE TILDE, LATIN SMALL LETTER
+	0x026D: 'l', //  WITH RETROFLEX HOOK, LATIN SMALL LETTER
+	0x0142: 'l', //  WITH STROKE, LATIN SMALL LETTER
+	0x2097: 'l', // , LATIN SUBSCRIPT SMALL LETTER
+	0x1E3F: 'm', //  WITH ACUTE, LATIN SMALL LETTER
+	0x1E41: 'm', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E43: 'm', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0271: 'm', //  WITH HOOK, LATIN SMALL LETTER
+	0x0270: 'm', //  WITH LONG LEG, LATIN SMALL LETTER TURNED
+	0x036B: 'm', // , COMBINING LATIN SMALL LETTER
+	0x1D1F: 'm', // , LATIN SMALL LETTER SIDEWAYS TURNED
+	0x026F: 'm', // , LATIN SMALL LETTER TURNED
+	0x2098: 'm', // , LATIN SUBSCRIPT SMALL LETTER
+	0x0144: 'n', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0148: 'n', //  WITH CARON, LATIN SMALL LETTER
+	0x0146: 'n', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E4B: 'n', //  WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
+	0x0235: 'n', //  WITH CURL, LATIN SMALL LETTER
+	0x1E45: 'n', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E47: 'n', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x01F9: 'n', //  WITH GRAVE, LATIN SMALL LETTER
+	0x0272: 'n', //  WITH LEFT HOOK, LATIN SMALL LETTER
+	0x1E49: 'n', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x019E: 'n', //  WITH LONG RIGHT LEG, LATIN SMALL LETTER
+	0x0273: 'n', //  WITH RETROFLEX HOOK, LATIN SMALL LETTER
+	0x00F1: 'n', //  WITH TILDE, LATIN SMALL LETTER
+	0x2099: 'n', // , LATIN SUBSCRIPT SMALL LETTER
+	0x00F3: 'o', //  WITH ACUTE, LATIN SMALL LETTER
+	0x014F: 'o', //  WITH BREVE, LATIN SMALL LETTER
+	0x01D2: 'o', //  WITH CARON, LATIN SMALL LETTER
+	0x00F4: 'o', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x00F6: 'o', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x022F: 'o', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1ECD: 'o', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0151: 'o', //  WITH DOUBLE ACUTE, LATIN SMALL LETTER
+	0x020D: 'o', //  WITH DOUBLE GRAVE, LATIN SMALL LETTER
+	0x00F2: 'o', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1ECF: 'o', //  WITH HOOK ABOVE, LATIN SMALL LETTER
+	0x01A1: 'o', //  WITH HORN, LATIN SMALL LETTER
+	0x020F: 'o', //  WITH INVERTED BREVE, LATIN SMALL LETTER
+	0x014D: 'o', //  WITH MACRON, LATIN SMALL LETTER
+	0x01EB: 'o', //  WITH OGONEK, LATIN SMALL LETTER
+	0x00F8: 'o', //  WITH STROKE, LATIN SMALL LETTER
+	0x1D13: 'o', //  WITH STROKE, LATIN SMALL LETTER SIDEWAYS
+	0x00F5: 'o', //  WITH TILDE, LATIN SMALL LETTER
+	0x0366: 'o', // , COMBINING LATIN SMALL LETTER
+	0x0275: 'o', // , LATIN SMALL LETTER BARRED
+	0x1D17: 'o', // , LATIN SMALL LETTER BOTTOM HALF
+	0x0254: 'o', // , LATIN SMALL LETTER OPEN
+	0x1D11: 'o', // , LATIN SMALL LETTER SIDEWAYS
+	0x1D12: 'o', // , LATIN SMALL LETTER SIDEWAYS OPEN
+	0x1D16: 'o', // , LATIN SMALL LETTER TOP HALF
+	0x1E55: 'p', //  WITH ACUTE, LATIN SMALL LETTER
+	0x1E57: 'p', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x01A5: 'p', //  WITH HOOK, LATIN SMALL LETTER
+	0x209A: 'p', // , LATIN SUBSCRIPT SMALL LETTER
+	0x024B: 'q', //  WITH HOOK TAIL, LATIN SMALL LETTER
+	0x02A0: 'q', //  WITH HOOK, LATIN SMALL LETTER
+	0x0155: 'r', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0159: 'r', //  WITH CARON, LATIN SMALL LETTER
+	0x0157: 'r', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E59: 'r', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E5B: 'r', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0211: 'r', //  WITH DOUBLE GRAVE, LATIN SMALL LETTER
+	0x027E: 'r', //  WITH FISHHOOK, LATIN SMALL LETTER
+	0x027F: 'r', //  WITH FISHHOOK, LATIN SMALL LETTER REVERSED
+	0x027B: 'r', //  WITH HOOK, LATIN SMALL LETTER TURNED
+	0x0213: 'r', //  WITH INVERTED BREVE, LATIN SMALL LETTER
+	0x1E5F: 'r', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x027C: 'r', //  WITH LONG LEG, LATIN SMALL LETTER
+	0x027A: 'r', //  WITH LONG LEG, LATIN SMALL LETTER TURNED
+	0x024D: 'r', //  WITH STROKE, LATIN SMALL LETTER
+	0x027D: 'r', //  WITH TAIL, LATIN SMALL LETTER
+	0x036C: 'r', // , COMBINING LATIN SMALL LETTER
+	0x0279: 'r', // , LATIN SMALL LETTER TURNED
+	0x1D63: 'r', // , LATIN SUBSCRIPT SMALL LETTER
+	0x015B: 's', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0161: 's', //  WITH CARON, LATIN SMALL LETTER
+	0x015F: 's', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x015D: 's', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x0219: 's', //  WITH COMMA BELOW, LATIN SMALL LETTER
+	0x1E61: 's', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E9B: 's', //  WITH DOT ABOVE, LATIN SMALL LETTER LONG
+	0x1E63: 's', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0282: 's', //  WITH HOOK, LATIN SMALL LETTER
+	0x023F: 's', //  WITH SWASH TAIL, LATIN SMALL LETTER
+	0x017F: 's', // , LATIN SMALL LETTER LONG
+	0x00DF: 's', // , LATIN SMALL LETTER SHARP
+	0x209B: 's', // , LATIN SUBSCRIPT SMALL LETTER
+	0x0165: 't', //  WITH CARON, LATIN SMALL LETTER
+	0x0163: 't', //  WITH CEDILLA, LATIN SMALL LETTER
+	0x1E71: 't', //  WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
+	0x021B: 't', //  WITH COMMA BELOW, LATIN SMALL LETTER
+	0x0236: 't', //  WITH CURL, LATIN SMALL LETTER
+	0x1E97: 't', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1E6B: 't', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E6D: 't', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x01AD: 't', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E6F: 't', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x01AB: 't', //  WITH PALATAL HOOK, LATIN SMALL LETTER
+	0x0288: 't', //  WITH RETROFLEX HOOK, LATIN SMALL LETTER
+	0x0167: 't', //  WITH STROKE, LATIN SMALL LETTER
+	0x036D: 't', // , COMBINING LATIN SMALL LETTER
+	0x0287: 't', // , LATIN SMALL LETTER TURNED
+	0x209C: 't', // , LATIN SUBSCRIPT SMALL LETTER
+	0x0289: 'u', //  BAR, LATIN SMALL LETTER
+	0x00FA: 'u', //  WITH ACUTE, LATIN SMALL LETTER
+	0x016D: 'u', //  WITH BREVE, LATIN SMALL LETTER
+	0x01D4: 'u', //  WITH CARON, LATIN SMALL LETTER
+	0x1E77: 'u', //  WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
+	0x00FB: 'u', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x1E73: 'u', //  WITH DIAERESIS BELOW, LATIN SMALL LETTER
+	0x00FC: 'u', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1EE5: 'u', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0171: 'u', //  WITH DOUBLE ACUTE, LATIN SMALL LETTER
+	0x0215: 'u', //  WITH DOUBLE GRAVE, LATIN SMALL LETTER
+	0x00F9: 'u', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1EE7: 'u', //  WITH HOOK ABOVE, LATIN SMALL LETTER
+	0x01B0: 'u', //  WITH HORN, LATIN SMALL LETTER
+	0x0217: 'u', //  WITH INVERTED BREVE, LATIN SMALL LETTER
+	0x016B: 'u', //  WITH MACRON, LATIN SMALL LETTER
+	0x0173: 'u', //  WITH OGONEK, LATIN SMALL LETTER
+	0x016F: 'u', //  WITH RING ABOVE, LATIN SMALL LETTER
+	0x1E75: 'u', //  WITH TILDE BELOW, LATIN SMALL LETTER
+	0x0169: 'u', //  WITH TILDE, LATIN SMALL LETTER
+	0x0367: 'u', // , COMBINING LATIN SMALL LETTER
+	0x1D1D: 'u', // , LATIN SMALL LETTER SIDEWAYS
+	0x1D1E: 'u', // , LATIN SMALL LETTER SIDEWAYS DIAERESIZED
+	0x1D64: 'u', // , LATIN SUBSCRIPT SMALL LETTER
+	0x1E7F: 'v', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x028B: 'v', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E7D: 'v', //  WITH TILDE, LATIN SMALL LETTER
+	0x036E: 'v', // , COMBINING LATIN SMALL LETTER
+	0x028C: 'v', // , LATIN SMALL LETTER TURNED
+	0x1D65: 'v', // , LATIN SUBSCRIPT SMALL LETTER
+	0x1E83: 'w', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0175: 'w', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x1E85: 'w', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1E87: 'w', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E89: 'w', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x1E81: 'w', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1E98: 'w', //  WITH RING ABOVE, LATIN SMALL LETTER
+	0x028D: 'w', // , LATIN SMALL LETTER TURNED
+	0x1E8D: 'x', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1E8B: 'x', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x036F: 'x', // , COMBINING LATIN SMALL LETTER
+	0x00FD: 'y', //  WITH ACUTE, LATIN SMALL LETTER
+	0x0177: 'y', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x00FF: 'y', //  WITH DIAERESIS, LATIN SMALL LETTER
+	0x1E8F: 'y', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1EF5: 'y', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x1EF3: 'y', //  WITH GRAVE, LATIN SMALL LETTER
+	0x1EF7: 'y', //  WITH HOOK ABOVE, LATIN SMALL LETTER
+	0x01B4: 'y', //  WITH HOOK, LATIN SMALL LETTER
+	0x0233: 'y', //  WITH MACRON, LATIN SMALL LETTER
+	0x1E99: 'y', //  WITH RING ABOVE, LATIN SMALL LETTER
+	0x024F: 'y', //  WITH STROKE, LATIN SMALL LETTER
+	0x1EF9: 'y', //  WITH TILDE, LATIN SMALL LETTER
+	0x028E: 'y', // , LATIN SMALL LETTER TURNED
+	0x017A: 'z', //  WITH ACUTE, LATIN SMALL LETTER
+	0x017E: 'z', //  WITH CARON, LATIN SMALL LETTER
+	0x1E91: 'z', //  WITH CIRCUMFLEX, LATIN SMALL LETTER
+	0x0291: 'z', //  WITH CURL, LATIN SMALL LETTER
+	0x017C: 'z', //  WITH DOT ABOVE, LATIN SMALL LETTER
+	0x1E93: 'z', //  WITH DOT BELOW, LATIN SMALL LETTER
+	0x0225: 'z', //  WITH HOOK, LATIN SMALL LETTER
+	0x1E95: 'z', //  WITH LINE BELOW, LATIN SMALL LETTER
+	0x0290: 'z', //  WITH RETROFLEX HOOK, LATIN SMALL LETTER
+	0x01B6: 'z', //  WITH STROKE, LATIN SMALL LETTER
+	0x0240: 'z', //  WITH SWASH TAIL, LATIN SMALL LETTER
+	0x0251: 'a', // , latin small letter script
+	0x00C1: 'A', //  WITH ACUTE, LATIN CAPITAL LETTER
+	0x00C2: 'A', //  WITH CIRCUMFLEX, LATIN CAPITAL LETTER
+	0x00C4: 'A', //  WITH DIAERESIS, LATIN CAPITAL LETTER
+	0x00C0: 'A', //  WITH GRAVE, LATIN CAPITAL LETTER
+	0x00C5: 'A', //  WITH RING ABOVE, LATIN CAPITAL LETTER
+	0x023A: 'A', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x00C3: 'A', //  WITH TILDE, LATIN CAPITAL LETTER
+	0x1D00: 'A', // , LATIN LETTER SMALL CAPITAL
+	0x0181: 'B', //  WITH HOOK, LATIN CAPITAL LETTER
+	0x0243: 'B', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x0299: 'B', // , LATIN LETTER SMALL CAPITAL
+	0x1D03: 'B', // , LATIN LETTER SMALL CAPITAL BARRED
+	0x00C7: 'C', //  WITH CEDILLA, LATIN CAPITAL LETTER
+	0x023B: 'C', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x1D04: 'C', // , LATIN LETTER SMALL CAPITAL
+	0x018A: 'D', //  WITH HOOK, LATIN CAPITAL LETTER
+	0x0189: 'D', // , LATIN CAPITAL LETTER AFRICAN
+	0x1D05: 'D', // , LATIN LETTER SMALL CAPITAL
+	0x00C9: 'E', //  WITH ACUTE, LATIN CAPITAL LETTER
+	0x00CA: 'E', //  WITH CIRCUMFLEX, LATIN CAPITAL LETTER
+	0x00CB: 'E', //  WITH DIAERESIS, LATIN CAPITAL LETTER
+	0x00C8: 'E', //  WITH GRAVE, LATIN CAPITAL LETTER
+	0x0246: 'E', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x0190: 'E', // , LATIN CAPITAL LETTER OPEN
+	0x018E: 'E', // , LATIN CAPITAL LETTER REVERSED
+	0x1D07: 'E', // , LATIN LETTER SMALL CAPITAL
+	0x0193: 'G', //  WITH HOOK, LATIN CAPITAL LETTER
+	0x029B: 'G', //  WITH HOOK, LATIN LETTER SMALL CAPITAL
+	0x0262: 'G', // , LATIN LETTER SMALL CAPITAL
+	0x029C: 'H', // , LATIN LETTER SMALL CAPITAL
+	0x00CD: 'I', //  WITH ACUTE, LATIN CAPITAL LETTER
+	0x00CE: 'I', //  WITH CIRCUMFLEX, LATIN CAPITAL LETTER
+	0x00CF: 'I', //  WITH DIAERESIS, LATIN CAPITAL LETTER
+	0x0130: 'I', //  WITH DOT ABOVE, LATIN CAPITAL LETTER
+	0x00CC: 'I', //  WITH GRAVE, LATIN CAPITAL LETTER
+	0x0197: 'I', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x026A: 'I', // , LATIN LETTER SMALL CAPITAL
+	0x0248: 'J', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x1D0A: 'J', // , LATIN LETTER SMALL CAPITAL
+	0x1D0B: 'K', // , LATIN LETTER SMALL CAPITAL
+	0x023D: 'L', //  WITH BAR, LATIN CAPITAL LETTER
+	0x1D0C: 'L', //  WITH STROKE, LATIN LETTER SMALL CAPITAL
+	0x029F: 'L', // , LATIN LETTER SMALL CAPITAL
+	0x019C: 'M', // , LATIN CAPITAL LETTER TURNED
+	0x1D0D: 'M', // , LATIN LETTER SMALL CAPITAL
+	0x019D: 'N', //  WITH LEFT HOOK, LATIN CAPITAL LETTER
+	0x0220: 'N', //  WITH LONG RIGHT LEG, LATIN CAPITAL LETTER
+	0x00D1: 'N', //  WITH TILDE, LATIN CAPITAL LETTER
+	0x0274: 'N', // , LATIN LETTER SMALL CAPITAL
+	0x1D0E: 'N', // , LATIN LETTER SMALL CAPITAL REVERSED
+	0x00D3: 'O', //  WITH ACUTE, LATIN CAPITAL LETTER
+	0x00D4: 'O', //  WITH CIRCUMFLEX, LATIN CAPITAL LETTER
+	0x00D6: 'O', //  WITH DIAERESIS, LATIN CAPITAL LETTER
+	0x00D2: 'O', //  WITH GRAVE, LATIN CAPITAL LETTER
+	0x019F: 'O', //  WITH MIDDLE TILDE, LATIN CAPITAL LETTER
+	0x00D8: 'O', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x00D5: 'O', //  WITH TILDE, LATIN CAPITAL LETTER
+	0x0186: 'O', // , LATIN CAPITAL LETTER OPEN
+	0x1D0F: 'O', // , LATIN LETTER SMALL CAPITAL
+	0x1D10: 'O', // , LATIN LETTER SMALL CAPITAL OPEN
+	0x1D18: 'P', // , LATIN LETTER SMALL CAPITAL
+	0x024A: 'Q', //  WITH HOOK TAIL, LATIN CAPITAL LETTER SMALL
+	0x024C: 'R', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x0280: 'R', // , LATIN LETTER SMALL CAPITAL
+	0x0281: 'R', // , LATIN LETTER SMALL CAPITAL INVERTED
+	0x1D19: 'R', // , LATIN LETTER SMALL CAPITAL REVERSED
+	0x1D1A: 'R', // , LATIN LETTER SMALL CAPITAL TURNED
+	0x023E: 'T', //  WITH DIAGONAL STROKE, LATIN CAPITAL LETTER
+	0x01AE: 'T', //  WITH RETROFLEX HOOK, LATIN CAPITAL LETTER
+	0x1D1B: 'T', // , LATIN LETTER SMALL CAPITAL
+	0x0244: 'U', //  BAR, LATIN CAPITAL LETTER
+	0x00DA: 'U', //  WITH ACUTE, LATIN CAPITAL LETTER
+	0x00DB: 'U', //  WITH CIRCUMFLEX, LATIN CAPITAL LETTER
+	0x00DC: 'U', //  WITH DIAERESIS, LATIN CAPITAL LETTER
+	0x00D9: 'U', //  WITH GRAVE, LATIN CAPITAL LETTER
+	0x1D1C: 'U', // , LATIN LETTER SMALL CAPITAL
+	0x01B2: 'V', //  WITH HOOK, LATIN CAPITAL LETTER
+	0x0245: 'V', // , LATIN CAPITAL LETTER TURNED
+	0x1D20: 'V', // , LATIN LETTER SMALL CAPITAL
+	0x1D21: 'W', // , LATIN LETTER SMALL CAPITAL
+	0x00DD: 'Y', //  WITH ACUTE, LATIN CAPITAL LETTER
+	0x0178: 'Y', //  WITH DIAERESIS, LATIN CAPITAL LETTER
+	0x024E: 'Y', //  WITH STROKE, LATIN CAPITAL LETTER
+	0x028F: 'Y', // , LATIN LETTER SMALL CAPITAL
+	0x1D22: 'Z', // , LATIN LETTER SMALL CAPITAL
+}

src/core.go

@@ -143,7 +143,7 @@ func Run(opts *Options) {
 	}
 	patternBuilder := func(runes []rune) *Pattern {
 		return BuildPattern(
-			opts.Fuzzy, opts.FuzzyAlgo, opts.Extended, opts.Case, forward,
+			opts.Fuzzy, opts.FuzzyAlgo, opts.Extended, opts.Case, opts.Normalize, forward,
 			opts.Filter == nil, opts.Nth, opts.Delimiter, runes)
 	}
 	matcher := NewMatcher(patternBuilder, sort, opts.Tac, eventBox)

src/options.go

@@ -24,6 +24,7 @@ const usage = `usage: fzf [options]
     --algo=TYPE           Fuzzy matching algorithm: [v1|v2] (default: v2)
     -i                    Case-insensitive match (default: smart-case match)
     +i                    Case-sensitive match
+    --normalize           Normalize latin script letters before matching
     -n, --nth=N[,..]      Comma-separated list of field index expressions
                           for limiting search scope. Each can be a non-zero
                           integer or a range expression ([BEGIN]..[END]).
@@ -138,6 +139,7 @@ type Options struct {
 	FuzzyAlgo   algo.Algo
 	Extended    bool
 	Case        Case
+	Normalize   bool
 	Nth         []Range
 	WithNth     []Range
 	Delimiter   Delimiter
@@ -185,6 +187,7 @@ func defaultOptions() *Options {
 		FuzzyAlgo:   algo.FuzzyMatchV2,
 		Extended:    true,
 		Case:        CaseSmart,
+		Normalize:   false,
 		Nth:         make([]Range, 0),
 		WithNth:     make([]Range, 0),
 		Delimiter:   Delimiter{},
@@ -887,6 +890,10 @@ func parseOptions(opts *Options, allArgs []string) {
 		case "-f", "--filter":
 			filter := nextString(allArgs, &i, "query string required")
 			opts.Filter = &filter
+		case "--normalize":
+			opts.Normalize = true
+		case "--no-normalize":
+			opts.Normalize = false
 		case "--algo":
 			opts.FuzzyAlgo = parseAlgo(nextString(allArgs, &i, "algorithm required (v1|v2)"))
 		case "--expect":

src/pattern.go

@@ -43,6 +43,7 @@ type Pattern struct {
 	fuzzyAlgo     algo.Algo
 	extended      bool
 	caseSensitive bool
+	normalize     bool
 	forward       bool
 	text          []rune
 	termSets      []termSet
@@ -75,7 +76,7 @@ func clearChunkCache() {
 }
 
 // BuildPattern builds Pattern object from the given arguments
-func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, forward bool,
+func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, normalize bool, forward bool,
 	cacheable bool, nth []Range, delimiter Delimiter, runes []rune) *Pattern {
 
 	var asString string
@@ -120,6 +121,7 @@ func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case,
 		fuzzyAlgo:     fuzzyAlgo,
 		extended:      extended,
 		caseSensitive: caseSensitive,
+		normalize:     normalize,
 		forward:       forward,
 		text:          []rune(asString),
 		termSets:      termSets,
@@ -309,9 +311,9 @@ func (p *Pattern) MatchItem(item *Item, withPos bool, slab *util.Slab) (*Result,
 func (p *Pattern) basicMatch(item *Item, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) {
 	input := p.prepareInput(item)
 	if p.fuzzy {
-		return p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.forward, p.text, withPos, slab)
+		return p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)
 	}
-	return p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.forward, p.text, withPos, slab)
+	return p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)
 }
 
 func (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Offset, int, int, *[]int) {
@@ -330,7 +332,7 @@ func (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Of
 		matched := false
 		for _, term := range termSet {
 			pfun := p.procFun[term.typ]
-			off, score, tLen, pos := p.iter(pfun, input, term.caseSensitive, p.forward, term.text, withPos, slab)
+			off, score, tLen, pos := p.iter(pfun, input, term.caseSensitive, p.normalize, p.forward, term.text, withPos, slab)
 			if sidx := off[0]; sidx >= 0 {
 				if term.inv {
 					continue
@@ -378,9 +380,9 @@ func (p *Pattern) prepareInput(item *Item) []Token {
 	return ret
 }
 
-func (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) {
+func (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, normalize bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) {
 	for _, part := range tokens {
-		if res, pos := pfun(caseSensitive, forward, *part.text, pattern, withPos, slab); res.Start >= 0 {
+		if res, pos := pfun(caseSensitive, normalize, forward, *part.text, pattern, withPos, slab); res.Start >= 0 {
 			sidx := int32(res.Start) + part.prefixLength
 			eidx := int32(res.End) + part.prefixLength
 			if pos != nil {

src/pattern_test.go

@@ -75,10 +75,10 @@ func TestParseTermsEmpty(t *testing.T) {
 func TestExact(t *testing.T) {
 	defer clearPatternCache()
 	clearPatternCache()
-	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true,
+	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true,
 		[]Range{}, Delimiter{}, []rune("'abc"))
 	res, pos := algo.ExactMatchNaive(
-		pattern.caseSensitive, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil)
+		pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil)
 	if res.Start != 7 || res.End != 10 {
 		t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End)
 	}
@@ -90,11 +90,11 @@ func TestExact(t *testing.T) {
 func TestEqual(t *testing.T) {
 	defer clearPatternCache()
 	clearPatternCache()
-	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("^AbC$"))
+	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("^AbC$"))
 
 	match := func(str string, sidxExpected int, eidxExpected int) {
 		res, pos := algo.EqualMatch(
-			pattern.caseSensitive, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil)
+			pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil)
 		if res.Start != sidxExpected || res.End != eidxExpected {
 			t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End)
 		}
@@ -109,17 +109,17 @@ func TestEqual(t *testing.T) {
 func TestCaseSensitivity(t *testing.T) {
 	defer clearPatternCache()
 	clearPatternCache()
-	pat1 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("abc"))
+	pat1 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("abc"))
 	clearPatternCache()
-	pat2 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("Abc"))
+	pat2 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("Abc"))
 	clearPatternCache()
-	pat3 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, true, true, []Range{}, Delimiter{}, []rune("abc"))
+	pat3 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, false, true, true, []Range{}, Delimiter{}, []rune("abc"))
 	clearPatternCache()
-	pat4 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, true, true, []Range{}, Delimiter{}, []rune("Abc"))
+	pat4 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, false, true, true, []Range{}, Delimiter{}, []rune("Abc"))
 	clearPatternCache()
-	pat5 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, true, true, []Range{}, Delimiter{}, []rune("abc"))
+	pat5 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, false, true, true, []Range{}, Delimiter{}, []rune("abc"))
 	clearPatternCache()
-	pat6 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, true, true, []Range{}, Delimiter{}, []rune("Abc"))
+	pat6 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, false, true, true, []Range{}, Delimiter{}, []rune("Abc"))
 
 	if string(pat1.text) != "abc" || pat1.caseSensitive != false ||
 		string(pat2.text) != "Abc" || pat2.caseSensitive != true ||
@@ -132,7 +132,7 @@ func TestCaseSensitivity(t *testing.T) {
 }
 
 func TestOrigTextAndTransformed(t *testing.T) {
-	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("jg"))
+	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("jg"))
 	tokens := Tokenize(util.RunesToChars([]rune("junegunn")), Delimiter{})
 	trans := Transform(tokens, []Range{Range{1, 1}})
 
@@ -167,7 +167,7 @@ func TestOrigTextAndTransformed(t *testing.T) {
 
 func TestCacheKey(t *testing.T) {
 	test := func(extended bool, patStr string, expected string, cacheable bool) {
-		pat := BuildPattern(true, algo.FuzzyMatchV2, extended, CaseSmart, true, true, []Range{}, Delimiter{}, []rune(patStr))
+		pat := BuildPattern(true, algo.FuzzyMatchV2, extended, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune(patStr))
 		if pat.CacheKey() != expected {
 			t.Errorf("Expected: %s, actual: %s", expected, pat.CacheKey())
 		}