diff --git a/src/chunklist.go b/src/chunklist.go index c20ffd4..a953fae 100644 --- a/src/chunklist.go +++ b/src/chunklist.go @@ -7,7 +7,7 @@ type Chunk []*Item // >>> []Item // ItemBuilder is a closure type that builds Item object from a pointer to a // string and an integer -type ItemBuilder func([]rune, int) *Item +type ItemBuilder func([]byte, int) *Item // ChunkList is a list of Chunks type ChunkList struct { @@ -26,7 +26,7 @@ func NewChunkList(trans ItemBuilder) *ChunkList { trans: trans} } -func (c *Chunk) push(trans ItemBuilder, data []rune, index int) bool { +func (c *Chunk) push(trans ItemBuilder, data []byte, index int) bool { item := trans(data, index) if item != nil { *c = append(*c, item) @@ -53,7 +53,7 @@ func CountItems(cs []*Chunk) int { } // Push adds the item to the list -func (cl *ChunkList) Push(data []rune) bool { +func (cl *ChunkList) Push(data []byte) bool { cl.mutex.Lock() defer cl.mutex.Unlock() diff --git a/src/chunklist_test.go b/src/chunklist_test.go index faaf04f..26795ef 100644 --- a/src/chunklist_test.go +++ b/src/chunklist_test.go @@ -6,8 +6,8 @@ import ( ) func TestChunkList(t *testing.T) { - cl := NewChunkList(func(s []rune, i int) *Item { - return &Item{text: s, rank: Rank{0, 0, uint32(i * 2)}} + cl := NewChunkList(func(s []byte, i int) *Item { + return &Item{text: []rune(string(s)), rank: Rank{0, 0, uint32(i * 2)}} }) // Snapshot @@ -17,8 +17,8 @@ func TestChunkList(t *testing.T) { } // Add some data - cl.Push([]rune("hello")) - cl.Push([]rune("world")) + cl.Push([]byte("hello")) + cl.Push([]byte("world")) // Previously created snapshot should remain the same if len(snapshot) > 0 { @@ -46,7 +46,7 @@ func TestChunkList(t *testing.T) { // Add more data for i := 0; i < chunkSize*2; i++ { - cl.Push([]rune(fmt.Sprintf("item %d", i))) + cl.Push([]byte(fmt.Sprintf("item %d", i))) } // Previous snapshot should remain the same @@ -64,8 +64,8 @@ func TestChunkList(t *testing.T) { t.Error("Unexpected number of items") } - cl.Push([]rune("hello")) - cl.Push([]rune("world")) + cl.Push([]byte("hello")) + cl.Push([]byte("world")) lastChunkCount := len(*snapshot[len(snapshot)-1]) if lastChunkCount != 2 { diff --git a/src/core.go b/src/core.go index c0596e3..fdd1e06 100644 --- a/src/core.go +++ b/src/core.go @@ -63,48 +63,54 @@ func Run(opts *Options) { eventBox := util.NewEventBox() // ANSI code processor - ansiProcessor := func(runes []rune) ([]rune, []ansiOffset) { - // By default, we do nothing - return runes, nil + ansiProcessor := func(data []byte) ([]rune, []ansiOffset) { + return util.BytesToRunes(data), nil + } + ansiProcessorRunes := func(data []rune) ([]rune, []ansiOffset) { + return data, nil } if opts.Ansi { if opts.Theme != nil { var state *ansiState - ansiProcessor = func(runes []rune) ([]rune, []ansiOffset) { - trimmed, offsets, newState := extractColor(string(runes), state) + ansiProcessor = func(data []byte) ([]rune, []ansiOffset) { + trimmed, offsets, newState := extractColor(string(data), state) state = newState return []rune(trimmed), offsets } } else { // When color is disabled but ansi option is given, // we simply strip out ANSI codes from the input - ansiProcessor = func(runes []rune) ([]rune, []ansiOffset) { - trimmed, _, _ := extractColor(string(runes), nil) + ansiProcessor = func(data []byte) ([]rune, []ansiOffset) { + trimmed, _, _ := extractColor(string(data), nil) return []rune(trimmed), nil } } + ansiProcessorRunes = func(data []rune) ([]rune, []ansiOffset) { + return ansiProcessor([]byte(string(data))) + } } // Chunk list var chunkList *ChunkList header := make([]string, 0, opts.HeaderLines) if len(opts.WithNth) == 0 { - chunkList = NewChunkList(func(data []rune, index int) *Item { + chunkList = NewChunkList(func(data []byte, index int) *Item { if len(header) < opts.HeaderLines { header = append(header, string(data)) eventBox.Set(EvtHeader, header) return nil } - data, colors := ansiProcessor(data) + runes, colors := ansiProcessor(data) return &Item{ - text: data, + text: runes, index: uint32(index), colors: colors, rank: Rank{0, 0, uint32(index)}} }) } else { - chunkList = NewChunkList(func(data []rune, index int) *Item { - tokens := Tokenize(data, opts.Delimiter) + chunkList = NewChunkList(func(data []byte, index int) *Item { + runes := util.BytesToRunes(data) + tokens := Tokenize(runes, opts.Delimiter) trans := Transform(tokens, opts.WithNth) if len(header) < opts.HeaderLines { header = append(header, string(joinTokens(trans))) @@ -113,12 +119,12 @@ func Run(opts *Options) { } item := Item{ text: joinTokens(trans), - origText: &data, + origText: &runes, index: uint32(index), colors: nil, rank: Rank{0, 0, uint32(index)}} - trimmed, colors := ansiProcessor(item.text) + trimmed, colors := ansiProcessorRunes(item.text) item.text = trimmed item.colors = colors return &item @@ -128,7 +134,7 @@ func Run(opts *Options) { // Reader streamingFilter := opts.Filter != nil && !sort && !opts.Tac && !opts.Sync if !streamingFilter { - reader := Reader{func(data []rune) bool { + reader := Reader{func(data []byte) bool { return chunkList.Push(data) }, eventBox, opts.ReadZero} go reader.ReadSource() @@ -151,7 +157,7 @@ func Run(opts *Options) { if streamingFilter { reader := Reader{ - func(runes []rune) bool { + func(runes []byte) bool { item := chunkList.trans(runes, 0) if item != nil && pattern.MatchItem(item) { fmt.Println(string(item.text)) diff --git a/src/reader.go b/src/reader.go index d979eb6..3e2cf0a 100644 --- a/src/reader.go +++ b/src/reader.go @@ -5,14 +5,13 @@ import ( "io" "os" "os/exec" - "unicode/utf8" "github.com/junegunn/fzf/src/util" ) // Reader reads from command or standard input type Reader struct { - pusher func([]rune) bool + pusher func([]byte) bool eventBox *util.EventBox delimNil bool } @@ -42,21 +41,10 @@ func (r *Reader) feed(src io.Reader) { // end in delim. bytea, err := reader.ReadBytes(delim) if len(bytea) > 0 { - runes := make([]rune, 0, len(bytea)) - for i := 0; i < len(bytea); { - if bytea[i] < utf8.RuneSelf { - runes = append(runes, rune(bytea[i])) - i++ - } else { - r, sz := utf8.DecodeRune(bytea[i:]) - i += sz - runes = append(runes, r) - } - } if err == nil { - runes = runes[:len(runes)-1] + bytea = bytea[:len(bytea)-1] } - if r.pusher(runes) { + if r.pusher(bytea) { r.eventBox.Set(EvtReadNew, nil) } } diff --git a/src/reader_test.go b/src/reader_test.go index bb68e51..d5c218c 100644 --- a/src/reader_test.go +++ b/src/reader_test.go @@ -10,7 +10,7 @@ func TestReadFromCommand(t *testing.T) { strs := []string{} eb := util.NewEventBox() reader := Reader{ - pusher: func(s []rune) bool { strs = append(strs, string(s)); return true }, + pusher: func(s []byte) bool { strs = append(strs, string(s)); return true }, eventBox: eb} // Check EventBox diff --git a/src/util/util.go b/src/util/util.go index a0e1269..eeeb75f 100644 --- a/src/util/util.go +++ b/src/util/util.go @@ -6,6 +6,7 @@ import "C" import ( "os" "time" + "unicode/utf8" ) // Max returns the largest integer @@ -88,3 +89,18 @@ func TrimRight(runes []rune) []rune { } return runes[0 : i+1] } + +func BytesToRunes(bytea []byte) []rune { + runes := make([]rune, 0, len(bytea)) + for i := 0; i < len(bytea); { + if bytea[i] < utf8.RuneSelf { + runes = append(runes, rune(bytea[i])) + i++ + } else { + r, sz := utf8.DecodeRune(bytea[i:]) + i += sz + runes = append(runes, r) + } + } + return runes +}