Further reduce unnecessary rune array conversion

I was too quick to release 0.16.9, this commit makes --ansi processing
even faster.

src/algo/algo_test.go

@@ -17,7 +17,7 @@ func assertMatch2(t *testing.T, fun Algo, caseSensitive, normalize, forward bool
 	if !caseSensitive {
 		pattern = strings.ToLower(pattern)
 	}
-	res, pos := fun(caseSensitive, normalize, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil)
+	res, pos := fun(caseSensitive, normalize, forward, util.ToChars([]byte(input)), []rune(pattern), true, nil)
 	var start, end int
 	if pos == nil || len(*pos) == 0 {
 		start = res.Start

src/core.go

@@ -69,14 +69,14 @@ func Run(opts *Options, revision string) {
 			ansiProcessor = func(data []byte) (util.Chars, *[]ansiOffset) {
 				trimmed, offsets, newState := extractColor(string(data), state, nil)
 				state = newState
-				return util.RunesToChars([]rune(trimmed)), offsets
+				return util.ToChars([]byte(trimmed)), offsets
 			}
 		} else {
 			// When color is disabled but ansi option is given,
 			// we simply strip out ANSI codes from the input
 			ansiProcessor = func(data []byte) (util.Chars, *[]ansiOffset) {
 				trimmed, _, _ := extractColor(string(data), nil, nil)
-				return util.RunesToChars([]rune(trimmed)), nil
+				return util.ToChars([]byte(trimmed)), nil
 			}
 		}
 	}

src/merger_test.go

@@ -17,7 +17,7 @@ func assert(t *testing.T, cond bool, msg ...string) {
 
 func randResult() Result {
 	str := fmt.Sprintf("%d", rand.Uint32())
-	chars := util.RunesToChars([]rune(str))
+	chars := util.ToChars([]byte(str))
 	chars.Index = rand.Int31()
 	return Result{item: &Item{text: chars}}
 }

src/pattern_test.go

@@ -78,7 +78,7 @@ func TestExact(t *testing.T) {
 	pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true,
 		[]Range{}, Delimiter{}, []rune("'abc"))
 	res, pos := algo.ExactMatchNaive(
-		pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil)
+		pattern.caseSensitive, pattern.normalize, pattern.forward, util.ToChars([]byte("aabbcc abc")), pattern.termSets[0][0].text, true, nil)
 	if res.Start != 7 || res.End != 10 {
 		t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End)
 	}
@@ -94,7 +94,7 @@ func TestEqual(t *testing.T) {
 
 	match := func(str string, sidxExpected int, eidxExpected int) {
 		res, pos := algo.EqualMatch(
-			pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil)
+			pattern.caseSensitive, pattern.normalize, pattern.forward, util.ToChars([]byte(str)), pattern.termSets[0][0].text, true, nil)
 		if res.Start != sidxExpected || res.End != eidxExpected {
 			t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End)
 		}
@@ -140,7 +140,7 @@ func TestOrigTextAndTransformed(t *testing.T) {
 	for _, extended := range []bool{false, true} {
 		chunk := Chunk{
 			Item{
-				text:        util.RunesToChars([]rune("junegunn")),
+				text:        util.ToChars([]byte("junegunn")),
 				origText:    &origBytes,
 				transformed: &trans},
 		}

src/terminal.go

@@ -712,7 +712,7 @@ func (t *Terminal) printHeader() {
 		trimmed, colors, newState := extractColor(lineStr, state, nil)
 		state = newState
 		item := &Item{
-			text:   util.RunesToChars([]rune(trimmed)),
+			text:   util.ToChars([]byte(trimmed)),
 			colors: colors}
 
 		t.move(line, 2, true)
@@ -1173,8 +1173,7 @@ func replacePlaceholder(template string, stripAnsi bool, delimiter Delimiter, fo
 		}
 
 		for idx, item := range items {
-			chars := util.RunesToChars([]rune(item.AsString(stripAnsi)))
+			tokens := Tokenize(item.AsString(stripAnsi), delimiter)
-			tokens := Tokenize(chars.ToString(), delimiter)
 			trans := Transform(tokens, ranges)
 			str := string(joinTokens(trans))
 			if delimiter.str != nil {

src/terminal_test.go

@@ -10,7 +10,7 @@ import (
 func newItem(str string) *Item {
 	bytes := []byte(str)
 	trimmed, _, _ := extractColor(str, nil, nil)
-	return &Item{origText: &bytes, text: util.RunesToChars([]rune(trimmed))}
+	return &Item{origText: &bytes, text: util.ToChars([]byte(trimmed))}
 }
 
 func TestReplacePlaceholder(t *testing.T) {