Auto merge of #2108 - puremourning:unicode, r=valloric

[READY] Fixes for multi-byte errors

# PR Prelude

Thank you for working on YCM! :)

**Please complete these steps and check these boxes (by putting an `x` inside
the brackets) _before_ filing your PR:**

- [X] I have read and understood YCM's [CONTRIBUTING][cont] document.
- [X] I have read and understood YCM's [CODE_OF_CONDUCT][code] document.
- [X] I have included tests for the changes in my PR. If not, I have included a
  rationale for why I haven't.
- [X] **I understand my PR may be closed if it becomes obvious I didn't
  actually perform all of these steps.**

# Why this change is necessary and useful

There are a number of recent errors with unicode (most of which caused by the server, see PR https://github.com/Valloric/ycmd/pull/455. In testing I fixed a number of client-side tracebacks also.

This is by no means a comprehensive set of fixes for the client - I have simply fixed those that I came across in testing.

Summary:
 - fixes for errors when typing in c-sharp files due to the completion done handler
 - fixes for FixIts to apply correctly with multi-byte characters
 - fixes for unicode characters in return from the omni completer

[cont]: https://github.com/Valloric/YouCompleteMe/blob/master/CONTRIBUTING.md
[code]: https://github.com/Valloric/YouCompleteMe/blob/master/CODE_OF_CONDUCT.md

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/youcompleteme/2108)
<!-- Reviewable:end -->

python/ycm/omni_completer.py

@@ -25,6 +25,7 @@ from builtins import *  # noqa
 
 import vim
 from ycm import vimsupport
+from ycmd import utils
 from ycmd.responses import ServerError
 from ycmd.completers.completer import Completer
 from ycm.client.base_request import BaseRequest, HandleServerException
@@ -65,8 +66,7 @@ class OmniCompleter( Completer ):
 
   def ComputeCandidates( self, request_data ):
     if self.ShouldUseCache():
-      return super( OmniCompleter, self ).ComputeCandidates(
+      return super( OmniCompleter, self ).ComputeCandidates( request_data )
-        request_data )
     else:
       if self.ShouldUseNowInner( request_data ):
         return self.ComputeCandidatesInner( request_data )
@@ -80,6 +80,7 @@ class OmniCompleter( Completer ):
     try:
       return_value = int( vim.eval( self._omnifunc + '(1,"")' ) )
       if return_value < 0:
+        # FIXME: Technically, if the return is -1 we should raise an error
         return []
 
       omnifunc_call = [ self._omnifunc,
@@ -89,12 +90,14 @@ class OmniCompleter( Completer ):
 
       items = vim.eval( ''.join( omnifunc_call ) )
 
-      if 'words' in items:
+      if isinstance( items, dict ) and 'words' in items:
         items = items[ 'words' ]
+
       if not hasattr( items, '__iter__' ):
         raise TypeError( OMNIFUNC_NOT_LIST )
 
       return list( filter( bool, items ) )
+
     except ( TypeError, ValueError, vim.error ) as error:
       vimsupport.PostVimMessage(
         OMNIFUNC_RETURNED_BAD_VALUE + ' ' + str( error ) )
@@ -102,7 +105,7 @@ class OmniCompleter( Completer ):
 
 
   def OnFileReadyToParse( self, request_data ):
-    self._omnifunc = vim.eval( '&omnifunc' )
+    self._omnifunc = utils.ToUnicode( vim.eval( '&omnifunc' ) )
 
 
   def FilterAndSortCandidatesInner( self, candidates, sort_property, query ):

python/ycm/test_utils.py

@@ -27,6 +27,10 @@ from mock import MagicMock
 from hamcrest import assert_that, equal_to
 import re
 import sys
+import nose
+import functools
+
+from ycmd.utils import ToUnicode
 
 
 BUFNR_REGEX = re.compile( r"^bufnr\('(.+)', ([0-9]+)\)$" )
@@ -43,6 +47,18 @@ BWIPEOUT_REGEX = re.compile( r"^(?:silent! )bwipeout!? ([0-9]+)$" )
 # https://github.com/Valloric/YouCompleteMe/pull/1694
 VIM_MOCK = MagicMock()
 
+# The default options which are only relevant to the client, not the server and
+# thus are not part of default_options.json, but are required for a working
+# YouCompleteMe or OmniCompleter object.
+DEFAULT_CLIENT_OPTIONS = {
+  'server_log_level': 'info',
+  'extra_conf_vim_data': [],
+  'show_diagnostics_ui': 1,
+  'enable_diagnostic_signs': 1,
+  'enable_diagnostic_highlighting': 0,
+  'always_populate_location_list': 0,
+}
+
 
 def MockGetBufferNumber( buffer_filename ):
   for buffer in VIM_MOCK.buffers:
@@ -126,7 +142,60 @@ def MockVimModule():
 
 
 class ExtendedMock( MagicMock ):
+  """An extension to the MagicMock class which adds the ability to check that a
+  callable is called with a precise set of calls in a precise order.
+
+  Example Usage:
+    from ycm.test_utils import ExtendedMock
+    @patch( 'test.testing', new_callable = ExtendedMock, ... )
+    def my_test( test_testing ):
+      ...
+  """
 
   def assert_has_exact_calls( self, calls, any_order = False ):
     self.assert_has_calls( calls, any_order )
     assert_that( self.call_count, equal_to( len( calls ) ) )
+
+
+def ExpectedFailure( reason, *exception_matchers ):
+  """Defines a decorator to be attached to tests. This decorator
+  marks the test as being known to fail, e.g. where documenting or exercising
+  known incorrect behaviour.
+
+  The parameters are:
+    - |reason| a textual description of the reason for the known issue. This
+               is used for the skip reason
+    - |exception_matchers| additional arguments are hamcrest matchers to apply
+                 to the exception thrown. If the matchers don't match, then the
+                 test is marked as error, with the original exception.
+
+  If the test fails (for the correct reason), then it is marked as skipped.
+  If it fails for any other reason, it is marked as failed.
+  If the test passes, then it is also marked as failed."""
+  def decorator( test ):
+    @functools.wraps( test )
+    def Wrapper( *args, **kwargs ):
+      try:
+        test( *args, **kwargs )
+      except Exception as test_exception:
+        # Ensure that we failed for the right reason
+        test_exception_message = ToUnicode( test_exception )
+        try:
+          for matcher in exception_matchers:
+            assert_that( test_exception_message, matcher )
+        except AssertionError:
+          # Failed for the wrong reason!
+          import traceback
+          print( 'Test failed for the wrong reason: ' + traceback.format_exc() )
+          # Real failure reason is the *original* exception, we're only trapping
+          # and ignoring the exception that is expected.
+          raise test_exception
+
+        # Failed for the right reason
+        raise nose.SkipTest( reason )
+      else:
+        raise AssertionError( 'Test was expected to fail: {0}'.format(
+          reason ) )
+    return Wrapper
+
+  return decorator

python/ycm/tests/event_notification_test.py

@@ -23,7 +23,7 @@ from future import standard_library
 standard_library.install_aliases()
 from builtins import *  # noqa
 
-from ycm.test_utils import MockVimModule, ExtendedMock
+from ycm.test_utils import MockVimModule, ExtendedMock, DEFAULT_CLIENT_OPTIONS
 MockVimModule()
 
 import contextlib
@@ -38,19 +38,6 @@ from mock import call, MagicMock, patch
 from nose.tools import eq_, ok_
 
 
-# The default options which are only relevant to the client, not the server and
-# thus are not part of default_options.json, but are required for a working
-# YouCompleteMe object.
-DEFAULT_CLIENT_OPTIONS = {
-  'server_log_level': 'info',
-  'extra_conf_vim_data': [],
-  'show_diagnostics_ui': 1,
-  'enable_diagnostic_signs': 1,
-  'enable_diagnostic_highlighting': 0,
-  'always_populate_location_list': 0,
-}
-
-
 def PostVimMessage_Call( message ):
   """Return a mock.call object for a call to vimsupport.PostVimMesasge with the
   supplied message"""

python/ycm/tests/omni_completer_test.py

@@ -0,0 +1,758 @@
+# encoding: utf-8
+#
+# Copyright (C) 2016 YouCompleteMe contributors
+#
+# This file is part of YouCompleteMe.
+#
+# YouCompleteMe is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# YouCompleteMe is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *  # noqa
+from future.utils import PY2
+
+from mock import patch, call
+from nose.tools import eq_
+from hamcrest import contains_string
+
+from ycm.test_utils import MockVimModule, ExtendedMock
+MockVimModule()
+
+from ycm.test_utils import DEFAULT_CLIENT_OPTIONS, ExpectedFailure
+from ycm.omni_completer import OmniCompleter
+from ycm.youcompleteme import YouCompleteMe
+
+from ycmd import user_options_store
+from ycmd.utils import ToBytes
+from ycmd.request_wrap import RequestWrap
+
+
+def ToBytesOnPY2( data ):
+  # To test the omnifunc, etc. returning strings, which can be of different
+  # types depending on python version, we use ToBytes on PY2 and just the native
+  # str on python3. This roughly matches what happens between py2 and py3
+  # versions within Vim
+  if PY2:
+    return ToBytes( data )
+
+  return data
+
+
+def BuildRequest( line_num, column_num, contents ):
+  # Note: it would be nice to use ycmd.test_utils.BuildRequest directly here,
+  # but we can't import ycmd.test_utils because that in turn imports ycm_core,
+  # which would cause our "ycm_core not imported" test to fail.
+  return {
+    'line_num': line_num,
+    'column_num': column_num,
+    'filepath': '/test',
+    'file_data': {
+      '/test': {
+        'contents': contents,
+        'filetypes': [ 'java' ] # We need a filetype with a trigger, so we just
+                                # use java
+      }
+    }
+  }
+
+
+def BuildRequestWrap( line_num, column_num, contents ):
+  return RequestWrap( BuildRequest( line_num, column_num, contents ) )
+
+
+def MakeUserOptions( custom_options = {} ):
+  options = dict( user_options_store.DefaultOptions() )
+  options.update( DEFAULT_CLIENT_OPTIONS )
+  options.update( custom_options )
+  return options
+
+
+class OmniCompleter_test( object ):
+
+  def setUp( self ):
+    # We need a server instance for FilterAndSortCandidates
+    self._server_state = YouCompleteMe( MakeUserOptions() )
+
+
+  def tearDown( self ):
+    self._server_state.OnVimLeave()
+
+
+  def OmniCompleter_GetCompletions_Cache_List_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = 'test.'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 6,
+                                     contents = contents )
+
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( 'a' ),
+                        ToBytesOnPY2( 'b' ),
+                        ToBytesOnPY2( 'cdef' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'')" ),
+      ] )
+
+      eq_( results, omnifunc_result )
+
+
+  def OmniCompleter_GetCompletions_Cache_ListFilter_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = 'test.t'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 7,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 't' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( 'a' ),
+                        ToBytesOnPY2( 'b' ),
+                        ToBytesOnPY2( 'cdef' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'t')" ),
+      ] )
+
+      eq_( results, [] )
+
+
+  def OmniCompleter_GetCompletions_NoCache_List_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 0
+    } ) )
+
+    contents = 'test.'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 6,
+                                     contents = contents )
+
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( 'a' ),
+                        ToBytesOnPY2( 'b' ),
+                        ToBytesOnPY2( 'cdef' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'')" ),
+      ] )
+
+      eq_( results, omnifunc_result )
+
+
+  def OmniCompleter_GetCompletions_NoCache_ListFilter_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 0
+    } ) )
+
+    contents = 'test.t'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 7,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 't' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( 'a' ),
+                        ToBytesOnPY2( 'b' ),
+                        ToBytesOnPY2( 'cdef' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'t')" ),
+      ] )
+
+      # actual result is that the results are not filtered, as we expect the
+      # omniufunc or vim itself to do this filtering
+      eq_( results, omnifunc_result )
+
+
+  @ExpectedFailure( 'We ignore the result of the call to findstart and use our '
+                    'own interpretation of where the identifier should be',
+                    contains_string( "test_omnifunc(0,'t')" ) )
+  def OmniCompleter_GetCompletsions_UseFindStart_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = 'test.t'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 7,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 't' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( 'a' ),
+                        ToBytesOnPY2( 'b' ),
+                        ToBytesOnPY2( 'cdef' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 1, omnifunc_result ] ) as vim_eval:
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+
+        # Fails here: actual result is that the findstart result (1) is ignored
+        # and we use the 't' query as we normally would on the server side
+        call( "test_omnifunc(0,'test.t')" ),
+      ] )
+
+      eq_( results, omnifunc_result )
+
+
+  def OmniCompleter_GetCompletions_Cache_Object_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = 'test.t'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 7,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 't' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = {
+      'words': [
+        ToBytesOnPY2( 'a' ),
+        ToBytesOnPY2( 'b' ),
+        ToBytesOnPY2( 'CDtEF' )
+      ]
+    }
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'t')" ),
+      ] )
+
+      eq_( results, [ ToBytesOnPY2( 'CDtEF' ) ] )
+
+
+  def OmniCompleter_GetCompletions_Cache_ObjectList_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = 'test.tt'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 8,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 'tt' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [
+      {
+        'word': ToBytesOnPY2( 'a' ),
+        'abbr': ToBytesOnPY2( 'ABBR'),
+        'menu': ToBytesOnPY2( 'MENU' ),
+        'info': ToBytesOnPY2( 'INFO' ),
+        'kind': ToBytesOnPY2( 'K' )
+      },
+      {
+        'word': ToBytesOnPY2( 'test' ),
+        'abbr': ToBytesOnPY2( 'ABBRTEST'),
+        'menu': ToBytesOnPY2( 'MENUTEST' ),
+        'info': ToBytesOnPY2( 'INFOTEST' ),
+        'kind': ToBytesOnPY2( 'T' )
+      }
+    ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'tt')" ),
+      ] )
+
+      eq_( results, [ omnifunc_result[ 1 ] ] )
+
+
+  def OmniCompleter_GetCompletions_NoCache_ObjectList_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 0
+    } ) )
+
+    contents = 'test.tt'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 8,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 'tt' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [
+      {
+        'word': ToBytesOnPY2( 'a' ),
+        'abbr': ToBytesOnPY2( 'ABBR'),
+        'menu': ToBytesOnPY2( 'MENU' ),
+        'info': ToBytesOnPY2( 'INFO' ),
+        'kind': ToBytesOnPY2( 'K' )
+      },
+      {
+        'word': ToBytesOnPY2( 'test' ),
+        'abbr': ToBytesOnPY2( 'ABBRTEST'),
+        'menu': ToBytesOnPY2( 'MENUTEST' ),
+        'info': ToBytesOnPY2( 'INFOTEST' ),
+        'kind': ToBytesOnPY2( 'T' )
+      }
+    ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'tt')" ),
+      ] )
+
+      # We don't filter the result - we expect the omnifunc to do that
+      # based on the query we supplied (Note: that means no fuzzy matching!)
+      eq_( results, omnifunc_result )
+
+
+  def OmniCompleter_GetCompletions_Cache_ObjectListObject_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = 'test.tt'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 8,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 'tt' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = {
+      'words': [
+        {
+          'word': ToBytesOnPY2( 'a' ),
+          'abbr': ToBytesOnPY2( 'ABBR'),
+          'menu': ToBytesOnPY2( 'MENU' ),
+          'info': ToBytesOnPY2( 'INFO' ),
+          'kind': ToBytesOnPY2( 'K' )
+        },
+        {
+          'word': ToBytesOnPY2( 'test' ),
+          'abbr': ToBytesOnPY2( 'ABBRTEST'),
+          'menu': ToBytesOnPY2( 'MENUTEST' ),
+          'info': ToBytesOnPY2( 'INFOTEST' ),
+          'kind': ToBytesOnPY2( 'T' )
+        }
+      ]
+    }
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'tt')" ),
+      ] )
+
+      eq_( results, [ omnifunc_result[ 'words' ][ 1 ] ] )
+
+
+  def OmniCompleter_GetCompletions_NoCache_ObjectListObject_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 0
+    } ) )
+
+    contents = 'test.tt'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 8,
+                                     contents = contents )
+
+    eq_( request_data[ 'query' ], 'tt' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = {
+      'words': [
+        {
+          'word': ToBytesOnPY2( 'a' ),
+          'abbr': ToBytesOnPY2( 'ABBR'),
+          'menu': ToBytesOnPY2( 'MENU' ),
+          'info': ToBytesOnPY2( 'INFO' ),
+          'kind': ToBytesOnPY2( 'K' )
+        },
+        {
+          'word': ToBytesOnPY2( 'test' ),
+          'abbr': ToBytesOnPY2( 'ABBRTEST'),
+          'menu': ToBytesOnPY2( 'MENUTEST' ),
+          'info': ToBytesOnPY2( 'INFOTEST' ),
+          'kind': ToBytesOnPY2( 'T' )
+        }
+      ]
+    }
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'tt')" ),
+      ] )
+
+      # No FilterAndSortCandidates for cache_omnifunc=0 (we expect the omnifunc
+      # to do the filtering?)
+      eq_( results, omnifunc_result[ 'words' ] )
+
+
+  def OmniCompleter_GetCompletions_Cache_List_Unicode_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = '†åsty_π.'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 13,
+                                     contents = contents )
+
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( '†est' ),
+                        ToBytesOnPY2( 'å_unicode_identifier' ),
+                        ToBytesOnPY2( 'πππππππ yummy πie' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'')" ),
+      ] )
+
+      eq_( results, omnifunc_result )
+
+
+  def OmniCompleter_GetCompletions_NoCache_List_Unicode_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 0
+    } ) )
+
+    contents = '†åsty_π.'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 13,
+                                     contents = contents )
+
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( '†est' ),
+                        ToBytesOnPY2( 'å_unicode_identifier' ),
+                        ToBytesOnPY2( 'πππππππ yummy πie' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'')" ),
+      ] )
+
+      eq_( results, omnifunc_result )
+
+
+  @ExpectedFailure( 'Filtering on unicode is not supported by the server' )
+  def OmniCompleter_GetCompletions_Cache_List_Filter_Unicode_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = '†åsty_π.ππ'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 17,
+                                     contents = contents )
+
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( '†est' ),
+                        ToBytesOnPY2( 'å_unicode_identifier' ),
+                        ToBytesOnPY2( 'πππππππ yummy πie' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'ππ')" ),
+      ] )
+
+      # Fails here: Filtering on unicode is not supported
+      eq_( results, [ omnifunc_result[ 2 ] ] )
+
+
+  def OmniCompleter_GetCompletions_NoCache_List_Filter_Unicode_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 0
+    } ) )
+
+    contents = '†åsty_π.ππ'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 17,
+                                     contents = contents )
+
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [ ToBytesOnPY2( 'πππππππ yummy πie' ) ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'ππ')" ),
+      ] )
+
+      eq_( results, omnifunc_result )
+
+
+  @ExpectedFailure( 'Filtering on unicode is not supported by the server' )
+  def OmniCompleter_GetCompletions_Cache_ObjectList_Unicode_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = '†åsty_π.ππ'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 17,
+                                     contents = contents )
+
+
+    eq_( request_data[ 'query' ], 'ππ' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = [
+      {
+        'word': ToBytesOnPY2( 'ålpha∫et' ),
+        'abbr': ToBytesOnPY2( 'å∫∫®'),
+        'menu': ToBytesOnPY2( 'µ´~¨á' ),
+        'info': ToBytesOnPY2( '^~fo' ),
+        'kind': ToBytesOnPY2( '˚' )
+      },
+      {
+        'word': ToBytesOnPY2( 'π†´ß†π' ),
+        'abbr': ToBytesOnPY2( 'ÅııÂʉÍÊ'),
+        'menu': ToBytesOnPY2( '˜‰ˆËʉÍÊ' ),
+        'info': ToBytesOnPY2( 'ȈÏØʉÍÊ' ),
+        'kind': ToBytesOnPY2( 'Ê' )
+      }
+    ]
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'ππ')" ),
+      ] )
+
+      # Fails here: Filtering on unicode is not supported
+      eq_( results, [ omnifunc_result[ 1 ] ] )
+
+
+  def OmniCompleter_GetCompletions_Cache_ObjectListObject_Unicode_test( self ):
+    omni_completer = OmniCompleter( MakeUserOptions( {
+      'cache_omnifunc': 1
+    } ) )
+
+    contents = '†åsty_π.t'
+    request_data = BuildRequestWrap( line_num = 1,
+                                     column_num = 14,
+                                     contents = contents )
+
+
+    eq_( request_data[ 'query' ], 't' )
+
+    # Make sure there is an omnifunc set up.
+    with patch( 'vim.eval', return_value = ToBytesOnPY2( 'test_omnifunc' ) ):
+      omni_completer.OnFileReadyToParse( request_data )
+
+    omnifunc_result = {
+      'words': [
+        {
+          'word': ToBytesOnPY2( 'ålpha∫et' ),
+          'abbr': ToBytesOnPY2( 'å∫∫®'),
+          'menu': ToBytesOnPY2( 'µ´~¨á' ),
+          'info': ToBytesOnPY2( '^~fo' ),
+          'kind': ToBytesOnPY2( '˚' )
+        },
+        {
+          'word': ToBytesOnPY2( 'π†´ß†π' ),
+          'abbr': ToBytesOnPY2( 'ÅııÂʉÍÊ'),
+          'menu': ToBytesOnPY2( '˜‰ˆËʉÍÊ' ),
+          'info': ToBytesOnPY2( 'ȈÏØʉÍÊ' ),
+          'kind': ToBytesOnPY2( 'Ê' )
+        },
+        {
+          'word': ToBytesOnPY2( 'test' ),
+          'abbr': ToBytesOnPY2( 'ÅııÂʉÍÊ'),
+          'menu': ToBytesOnPY2( '˜‰ˆËʉÍÊ' ),
+          'info': ToBytesOnPY2( 'ȈÏØʉÍÊ' ),
+          'kind': ToBytesOnPY2( 'Ê' )
+        }
+      ]
+    }
+
+    # And get the completions
+    with patch( 'vim.eval',
+                new_callable = ExtendedMock,
+                side_effect = [ 6, omnifunc_result ] ) as vim_eval:
+
+      results = omni_completer.ComputeCandidates( request_data )
+
+      vim_eval.assert_has_exact_calls( [
+        call( 'test_omnifunc(1,"")' ),
+        call( "test_omnifunc(0,'t')" ),
+      ] )
+
+      # Note: the filtered results are all unicode objects (not bytes) because
+      # they are passed through the FilterAndSortCandidates machinery
+      # (via the server)
+      eq_( results, [ {
+        'word': 'test',
+        'abbr': 'ÅııÂʉÍÊ',
+        'menu': '˜‰ˆËʉÍÊ',
+        'info': 'ȈÏØʉÍÊ',
+        'kind': 'Ê'
+      } ] )

python/ycm/tests/postcomplete_tests.py

@@ -1,3 +1,5 @@
+# encoding: utf-8
+#
 # Copyright (C) 2015 YouCompleteMe contributors
 #
 # This file is part of YouCompleteMe.
@@ -31,6 +33,7 @@ from hamcrest import assert_that, empty
 from mock import MagicMock, DEFAULT, patch
 from nose.tools import eq_, ok_
 
+from ycmd.utils import ToBytes
 from ycm import vimsupport
 from ycm.youcompleteme import YouCompleteMe
 
@@ -40,11 +43,11 @@ def GetVariableValue_CompleteItemIs( word, abbr = None, menu = None,
   def Result( variable ):
     if variable == 'v:completed_item':
       return {
-        'word': word,
+        'word': ToBytes( word ),
-        'abbr': abbr,
+        'abbr': ToBytes( abbr ),
-        'menu': menu,
+        'menu': ToBytes( menu ),
-        'info': info,
+        'info': ToBytes( info ),
-        'kind': kind,
+        'kind': ToBytes( kind ),
       }
     return DEFAULT
   return MagicMock( side_effect = Result )
@@ -115,7 +118,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.GetVariableValue',
           GetVariableValue_CompleteItemIs( 'Test' ) )
   def FilterToCompletedCompletions_NewVim_MatchIsReturned_test( self, *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._FilterToMatchingCompletions( completions, False )
     eq_( list( result ), completions )
 
@@ -125,7 +128,7 @@ class PostComplete_test():
           GetVariableValue_CompleteItemIs( 'A' ) )
   def FilterToCompletedCompletions_NewVim_ShortTextDoesntRaise_test( self,
                                                                      *args ):
-    completions = [ BuildCompletion( 'AAA' ) ]
+    completions = [ BuildCompletion( insertion_text = 'AAA' ) ]
     self.ycm._FilterToMatchingCompletions( completions, False )
 
 
@@ -134,7 +137,7 @@ class PostComplete_test():
           GetVariableValue_CompleteItemIs( 'Test' ) )
   def FilterToCompletedCompletions_NewVim_ExactMatchIsReturned_test( self,
                                                                      *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._FilterToMatchingCompletions( completions, False )
     eq_( list( result ), completions )
 
@@ -144,15 +147,24 @@ class PostComplete_test():
           GetVariableValue_CompleteItemIs( '   Quote' ) )
   def FilterToCompletedCompletions_NewVim_NonMatchIsntReturned_test( self,
                                                                      *args ):
-    completions = [ BuildCompletion( 'A' ) ]
+    completions = [ BuildCompletion( insertion_text = 'A' ) ]
     result = self.ycm._FilterToMatchingCompletions( completions, False )
     assert_that( list( result ), empty() )
 
 
+  @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = True )
+  @patch( 'ycm.vimsupport.GetVariableValue',
+          GetVariableValue_CompleteItemIs( '†es†' ) )
+  def FilterToCompletedCompletions_NewVim_Unicode_test( self, *args ):
+    completions = [ BuildCompletion( insertion_text = '†es†' ) ]
+    result = self.ycm._FilterToMatchingCompletions( completions, False )
+    eq_( list( result ), completions )
+
+
   @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = False )
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Test' )
   def FilterToCompletedCompletions_OldVim_MatchIsReturned_test( self, *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._FilterToMatchingCompletions( completions, False )
     eq_( list( result ), completions )
 
@@ -161,7 +173,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'X' )
   def FilterToCompletedCompletions_OldVim_ShortTextDoesntRaise_test( self,
                                                                      *args ):
-    completions = [ BuildCompletion( 'AAA' ) ]
+    completions = [ BuildCompletion( insertion_text = 'AAA' ) ]
     self.ycm._FilterToMatchingCompletions( completions, False )
 
 
@@ -169,7 +181,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Test' )
   def FilterToCompletedCompletions_OldVim_ExactMatchIsReturned_test( self,
                                                                      *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._FilterToMatchingCompletions( completions, False )
     eq_( list( result ), completions )
 
@@ -178,7 +190,15 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Quote' )
   def FilterToCompletedCompletions_OldVim_NonMatchIsntReturned_test( self,
                                                                      *args ):
-    completions = [ BuildCompletion( 'A' ) ]
+    completions = [ BuildCompletion( insertion_text = 'A' ) ]
+    result = self.ycm._FilterToMatchingCompletions( completions, False )
+    assert_that( list( result ), empty() )
+
+
+  @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = False )
+  @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Uniçø∂¢' )
+  def FilterToCompletedCompletions_OldVim_Unicode_test( self, *args ):
+    completions = [ BuildCompletion( insertion_text = 'Uniçø∂¢' ) ]
     result = self.ycm._FilterToMatchingCompletions( completions, False )
     assert_that( list( result ), empty() )
 
@@ -187,7 +207,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Te' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_MatchIsReturned_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
                                                                 completions )
     eq_( result, True )
@@ -197,7 +217,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'X' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_ShortTextDoesntRaise_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( "AAA" ) ]
+    completions = [ BuildCompletion( insertion_text = "AAA" ) ]
     self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions )
 
 
@@ -205,7 +225,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Test' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_ExactMatchIsntReturned_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
                                                                 completions )
     eq_( result, False )
@@ -215,19 +235,29 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Quote' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_NonMatchIsntReturned_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( 'A' ) ]
+    completions = [ BuildCompletion( insertion_text = 'A' ) ]
     result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
                                                                 completions )
     eq_( result, False )
 
 
+  @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = False )
+  @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Uniç' )
+  def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_Unicode_test(
+    self, *args ):
+    completions = [ BuildCompletion( insertion_text = 'Uniçø∂¢' ) ]
+    result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
+                                                                completions )
+    eq_( result, True )
+
+
   @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = True )
   @patch( 'ycm.vimsupport.GetVariableValue',
           GetVariableValue_CompleteItemIs( 'Te' ) )
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Quote' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_MatchIsReturned_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
                                                                 completions )
     eq_( result, True )
@@ -239,7 +269,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Quote' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_ShortTextDoesntRaise_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( 'AAA' ) ]
+    completions = [ BuildCompletion( insertion_text = 'AAA' ) ]
     self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions )
 
 
@@ -249,7 +279,7 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Quote' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_ExactMatchIsntReturned_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( 'Test' ) ]
+    completions = [ BuildCompletion( insertion_text = 'Test' ) ]
     result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
                                                                 completions )
     eq_( result, False )
@@ -261,12 +291,24 @@ class PostComplete_test():
   @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = '   Quote' )
   def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_NonMatchIsntReturned_test( # noqa
     self, *args ):
-    completions = [ BuildCompletion( "A" ) ]
+    completions = [ BuildCompletion( insertion_text = "A" ) ]
     result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
                                                                 completions )
     eq_( result, False )
 
 
+  @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = True )
+  @patch( 'ycm.vimsupport.GetVariableValue',
+          GetVariableValue_CompleteItemIs( 'Uniç' ) )
+  @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Uniç' )
+  def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_Unicode_test(
+    self, *args ):
+    completions = [ BuildCompletion( insertion_text = "Uniçø∂¢" ) ]
+    result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText(
+                                                                completions )
+    eq_( result, True )
+
+
   def GetRequiredNamespaceImport_ReturnNoneForNoExtraData_test( self ):
     eq_( None, self.ycm._GetRequiredNamespaceImport( {} ) )
 

python/ycm/tests/vimsupport_test.py

@@ -32,15 +32,14 @@ from ycm import vimsupport
 from nose.tools import eq_
 from hamcrest import assert_that, calling, raises, none, has_entry
 from mock import MagicMock, call, patch
-from ycmd.utils import ToBytes
+from ycmd.utils import ToBytes, ToUnicode
 import os
 import json
 
 
 def ReplaceChunk_SingleLine_Repl_1_test():
   # Replace with longer range
-  #                  12345678901234567
+  result_buffer = [ ToBytes( "This is a string" ) ]
-  result_buffer = [ "This is a string" ]
   start, end = _BuildLocations( 1, 1, 1, 5 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -49,7 +48,7 @@ def ReplaceChunk_SingleLine_Repl_1_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "How long is a string" ], result_buffer )
+  eq_( [ ToBytes( "How long is a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 4 )
 
@@ -66,7 +65,7 @@ def ReplaceChunk_SingleLine_Repl_1_test():
   line_offset += new_line_offset
   char_offset += new_char_offset
 
-  eq_( [ 'How long is a piece of string' ], result_buffer )
+  eq_( [ ToBytes( 'How long is a piece of string' ) ], result_buffer )
   eq_( new_line_offset, 0 )
   eq_( new_char_offset, 9 )
   eq_( line_offset, 0 )
@@ -86,7 +85,7 @@ def ReplaceChunk_SingleLine_Repl_1_test():
   line_offset += new_line_offset
   char_offset += new_char_offset
 
-  eq_( ['How long is a piece of pie' ], result_buffer )
+  eq_( [ ToBytes( 'How long is a piece of pie' ) ], result_buffer )
   eq_( new_line_offset, 0 )
   eq_( new_char_offset, -3 )
   eq_( line_offset, 0 )
@@ -95,8 +94,7 @@ def ReplaceChunk_SingleLine_Repl_1_test():
 
 def ReplaceChunk_SingleLine_Repl_2_test():
   # Replace with shorter range
-  #                  12345678901234567
+  result_buffer = [ ToBytes( "This is a string" ) ]
-  result_buffer = [ "This is a string" ]
   start, end = _BuildLocations( 1, 11, 1, 17 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -105,15 +103,14 @@ def ReplaceChunk_SingleLine_Repl_2_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This is a test" ], result_buffer )
+  eq_( [ ToBytes( "This is a test" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, -2 )
 
 
 def ReplaceChunk_SingleLine_Repl_3_test():
   # Replace with equal range
-  #                  12345678901234567
+  result_buffer = [ ToBytes( "This is a string" ) ]
-  result_buffer = [ "This is a string" ]
   start, end = _BuildLocations( 1, 6, 1, 8 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -122,14 +119,14 @@ def ReplaceChunk_SingleLine_Repl_3_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This be a string" ], result_buffer )
+  eq_( [ ToBytes( "This be a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 0 )
 
 
 def ReplaceChunk_SingleLine_Add_1_test():
   # Insert at start
-  result_buffer = [ "is a string" ]
+  result_buffer = [ ToBytes( "is a string" ) ]
   start, end = _BuildLocations( 1, 1, 1, 1 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -138,14 +135,14 @@ def ReplaceChunk_SingleLine_Add_1_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This is a string" ], result_buffer )
+  eq_( [ ToBytes( "This is a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 5 )
 
 
 def ReplaceChunk_SingleLine_Add_2_test():
   # Insert at end
-  result_buffer = [ "This is a " ]
+  result_buffer = [ ToBytes( "This is a " ) ]
   start, end = _BuildLocations( 1, 11, 1, 11 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -154,14 +151,14 @@ def ReplaceChunk_SingleLine_Add_2_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This is a string" ], result_buffer )
+  eq_( [ ToBytes( "This is a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 6 )
 
 
 def ReplaceChunk_SingleLine_Add_3_test():
   # Insert in the middle
-  result_buffer = [ "This is a string" ]
+  result_buffer = [ ToBytes( "This is a string" ) ]
   start, end = _BuildLocations( 1, 8, 1, 8 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -170,14 +167,14 @@ def ReplaceChunk_SingleLine_Add_3_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This is not a string" ], result_buffer )
+  eq_( [ ToBytes( "This is not a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 4 )
 
 
 def ReplaceChunk_SingleLine_Del_1_test():
   # Delete from start
-  result_buffer = [ "This is a string" ]
+  result_buffer = [ ToBytes( "This is a string" ) ]
   start, end = _BuildLocations( 1, 1, 1, 6 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -186,14 +183,14 @@ def ReplaceChunk_SingleLine_Del_1_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "is a string" ], result_buffer )
+  eq_( [ ToBytes( "is a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, -5 )
 
 
 def ReplaceChunk_SingleLine_Del_2_test():
   # Delete from end
-  result_buffer = [ "This is a string" ]
+  result_buffer = [ ToBytes( "This is a string" ) ]
   start, end = _BuildLocations( 1, 10, 1, 18 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -202,14 +199,14 @@ def ReplaceChunk_SingleLine_Del_2_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This is a" ], result_buffer )
+  eq_( [ ToBytes( "This is a" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, -8 )
 
 
 def ReplaceChunk_SingleLine_Del_3_test():
   # Delete from middle
-  result_buffer = [ "This is not a string" ]
+  result_buffer = [ ToBytes( "This is not a string" ) ]
   start, end = _BuildLocations( 1, 9, 1, 13 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -218,33 +215,84 @@ def ReplaceChunk_SingleLine_Del_3_test():
                                                           0,
                                                           result_buffer )
 
-  eq_( [ "This is a string" ], result_buffer )
+  eq_( [ ToBytes( "This is a string" ) ], result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, -4 )
 
 
+def ReplaceChunk_SingleLine_Unicode_ReplaceUnicodeChars_test():
+  # Replace Unicode characters.
+  result_buffer = [ ToBytes( "This Uniçø∂‰ string is in the middle" ) ]
+  start, end = _BuildLocations( 1, 6, 1, 20 )
+  ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
+                                                          end,
+                                                          'Unicode ',
+                                                          0,
+                                                          0,
+                                                          result_buffer )
+
+  eq_( [ ToBytes( "This Unicode string is in the middle" ) ], result_buffer )
+  eq_( line_offset, 0 )
+  eq_( char_offset, -6 )
+
+
+def ReplaceChunk_SingleLine_Unicode_ReplaceAfterUnicode_test():
+  # Replace ASCII characters after Unicode characters in the line.
+  result_buffer = [ ToBytes( "This Uniçø∂‰ string is in the middle" ) ]
+  start, end = _BuildLocations( 1, 30, 1, 43 )
+  ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
+                                                          end,
+                                                          'fåke',
+                                                          0,
+                                                          0,
+                                                          result_buffer )
+
+  eq_( [ ToBytes( "This Uniçø∂‰ string is fåke" ) ], result_buffer )
+  eq_( line_offset, 0 )
+  eq_( char_offset, -8 )
+
+
+def ReplaceChunk_SingleLine_Unicode_Grown_test():
+  # Replace ASCII characters after Unicode characters in the line.
+  result_buffer = [ ToBytes( "a" ) ]
+  start, end = _BuildLocations( 1, 1, 1, 2 )
+  ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
+                                                          end,
+                                                          'å',
+                                                          0,
+                                                          0,
+                                                          result_buffer )
+
+  eq_( [ ToBytes( "å" ) ], result_buffer )
+  eq_( line_offset, 0 )
+  eq_( char_offset, 1 ) # Note: byte difference (a = 1 byte, å = 2 bytes)
+
+
 def ReplaceChunk_RemoveSingleLine_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 1, 3, 1 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, '',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, -1 )
   eq_( char_offset, 0 )
 
 
 def ReplaceChunk_SingleToMultipleLines_test():
-  result_buffer = [ "aAa",
+  result_buffer = [ ToBytes( "aAa" ),
-                    "aBa",
+                    ToBytes( "aBa" ),
-                    "aCa" ]
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 2, 2, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa",
+  expected_buffer = [ ToBytes( "aAa" ),
-                      "aEb",
+                      ToBytes( "aEb" ),
-                      "bFBa",
+                      ToBytes( "bFBa" ),
-                      "aCa" ]
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 1 )
   eq_( char_offset, 1 )
@@ -262,13 +310,16 @@ def ReplaceChunk_SingleToMultipleLines_test():
   line_offset += new_line_offset
   char_offset += new_char_offset
 
-  eq_( [ "aAa", "aEb", "bFBcccc", "aCa" ], result_buffer )
+  eq_( [ ToBytes( "aAa" ),
+         ToBytes( "aEb" ),
+         ToBytes( "bFBcccc" ),
+         ToBytes( "aCa" ) ], result_buffer )
   eq_( line_offset, 1 )
   eq_( char_offset, 4 )
 
 
 def ReplaceChunk_SingleToMultipleLines2_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 2, 2, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -276,14 +327,18 @@ def ReplaceChunk_SingleToMultipleLines2_test():
                                                           0,
                                                           0,
                                                           result_buffer )
-  expected_buffer = [ "aAa", "aEb", "bFb", "GBa", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aEb" ),
+                      ToBytes( "bFb" ),
+                      ToBytes( "GBa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 2 )
   eq_( char_offset, 0 )
 
 
 def ReplaceChunk_SingleToMultipleLines3_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 2, 2, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -291,14 +346,18 @@ def ReplaceChunk_SingleToMultipleLines3_test():
                                                           0,
                                                           0,
                                                           result_buffer )
-  expected_buffer = [ "aAa", "aEb", "bFb", "bGbBa", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aEb" ),
+                      ToBytes( "bFb" ),
+                      ToBytes( "bGbBa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 2 )
   eq_( char_offset, 2 )
 
 
 def ReplaceChunk_SingleToMultipleLinesReplace_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 1, 2, 1, 4 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -306,16 +365,20 @@ def ReplaceChunk_SingleToMultipleLinesReplace_test():
                                                           0,
                                                           0,
                                                           result_buffer )
-  expected_buffer = [ "aEb", "bFb", "bGb", "aBa", "aCa" ]
+  expected_buffer = [ ToBytes( "aEb" ),
+                      ToBytes( "bFb" ),
+                      ToBytes( "bGb" ),
+                      ToBytes( "aBa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 2 )
   eq_( char_offset, 0 )
 
 
 def ReplaceChunk_SingleToMultipleLinesReplace_2_test():
-  result_buffer = [ "aAa",
+  result_buffer = [ ToBytes( "aAa" ),
-                    "aBa",
+                    ToBytes( "aBa" ),
-                    "aCa" ]
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 1, 2, 1, 4 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -323,11 +386,11 @@ def ReplaceChunk_SingleToMultipleLinesReplace_2_test():
                                                           0,
                                                           0,
                                                           result_buffer )
-  expected_buffer = [ "aEb",
+  expected_buffer = [ ToBytes( "aEb" ),
-                      "bFb",
+                      ToBytes( "bFb" ),
-                      "bGb",
+                      ToBytes( "bGb" ),
-                      "aBa",
+                      ToBytes( "aBa" ),
-                      "aCa" ]
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 2 )
   eq_( char_offset, 0 )
@@ -345,22 +408,22 @@ def ReplaceChunk_SingleToMultipleLinesReplace_2_test():
   line_offset += new_line_offset
   char_offset += new_char_offset
 
-  eq_( [ "aEb",
+  eq_( [ ToBytes( "aEb" ),
-         "bFb",
+         ToBytes( "bFb" ),
-         "bGbcccc",
+         ToBytes( "bGbcccc" ),
-         "aBa",
+         ToBytes( "aBa" ),
-         "aCa" ], result_buffer )
+         ToBytes( "aCa" ) ], result_buffer )
 
   eq_( line_offset, 2 )
   eq_( char_offset, 4 )
 
 
 def ReplaceChunk_MultipleLinesToSingleLine_test():
-  result_buffer = [ "aAa", "aBa", "aCaaaa" ]
+  result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCaaaa" ) ]
   start, end = _BuildLocations( 2, 2, 3, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'E',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa", "aECaaaa" ]
+  expected_buffer = [ ToBytes( "aAa" ), ToBytes( "aECaaaa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, -1 )
   eq_( char_offset, 1 )
@@ -378,7 +441,8 @@ def ReplaceChunk_MultipleLinesToSingleLine_test():
   line_offset += new_line_offset
   char_offset += new_char_offset
 
-  eq_( [ "aAa", "aECccccaaa" ], result_buffer )
+  eq_( [ ToBytes( "aAa" ),
+         ToBytes( "aECccccaaa" ) ], result_buffer )
   eq_( line_offset, -1 )
   eq_( char_offset, 4 )
 
@@ -395,24 +459,36 @@ def ReplaceChunk_MultipleLinesToSingleLine_test():
   line_offset += new_line_offset
   char_offset += new_char_offset
 
-  eq_( [ "aAa", "aECccccdd", "ddaa" ], result_buffer )
+  eq_( [ ToBytes( "aAa" ),
+         ToBytes( "aECccccdd" ),
+         ToBytes( "ddaa" ) ],
+       result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, -2 )
 
 
 def ReplaceChunk_MultipleLinesToSameMultipleLines_test():
-  result_buffer = [ "aAa", "aBa", "aCa", "aDe" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ),
+                    ToBytes( "aDe" ) ]
   start, end = _BuildLocations( 2, 2, 3, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa", "aEb", "bFCa", "aDe" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aEb" ),
+                      ToBytes( "bFCa" ),
+                      ToBytes( "aDe" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 1 )
 
 
 def ReplaceChunk_MultipleLinesToMoreMultipleLines_test():
-  result_buffer = [ "aAa", "aBa", "aCa", "aDe" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ),
+                    ToBytes( "aDe" ) ]
   start, end = _BuildLocations( 2, 2, 3, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -420,113 +496,153 @@ def ReplaceChunk_MultipleLinesToMoreMultipleLines_test():
                                                           0,
                                                           0,
                                                           result_buffer )
-  expected_buffer = [ "aAa", "aEb", "bFb", "bGCa", "aDe" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aEb" ),
+                      ToBytes( "bFb" ),
+                      ToBytes( "bGCa" ),
+                      ToBytes( "aDe" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 1 )
   eq_( char_offset, 1 )
 
 
 def ReplaceChunk_MultipleLinesToLessMultipleLines_test():
-  result_buffer = [ "aAa", "aBa", "aCa", "aDe" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ),
+                    ToBytes( "aDe" ) ]
   start, end = _BuildLocations( 1, 2, 3, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aEb", "bFCa", "aDe" ]
+  expected_buffer = [ ToBytes( "aEb" ), ToBytes( "bFCa" ), ToBytes( "aDe" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, -1 )
   eq_( char_offset, 1 )
 
 
 def ReplaceChunk_MultipleLinesToEvenLessMultipleLines_test():
-  result_buffer = [ "aAa", "aBa", "aCa", "aDe" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ),
+                    ToBytes( "aDe" ) ]
   start, end = _BuildLocations( 1, 2, 4, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aEb", "bFDe" ]
+  expected_buffer = [ ToBytes( "aEb" ), ToBytes( "bFDe" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, -2 )
   eq_( char_offset, 1 )
 
 
 def ReplaceChunk_SpanBufferEdge_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 1, 1, 1, 3 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "bDba", "aBa", "aCa" ]
+  expected_buffer = [ ToBytes( "bDba" ),
+                      ToBytes( "aBa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 1 )
 
 
 def ReplaceChunk_DeleteTextInLine_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 2, 2, 3 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, '',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa", "aa", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, -1 )
 
 
 def ReplaceChunk_AddTextInLine_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 2, 2, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa", "abDbBa", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "abDbBa" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 3 )
 
 
 def ReplaceChunk_ReplaceTextInLine_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 2, 2, 2, 3 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb',
                                                           0, 0, result_buffer )
-  expected_buffer = [ "aAa", "abDba", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "abDba" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 2 )
 
 
 def ReplaceChunk_SingleLineOffsetWorks_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 1, 1, 1, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb',
                                                           1, 1, result_buffer )
-  expected_buffer = [ "aAa", "abDba", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "abDba" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 0 )
   eq_( char_offset, 2 )
 
 
 def ReplaceChunk_SingleLineToMultipleLinesOffsetWorks_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 1, 1, 1, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Db\nE',
                                                           1, 1, result_buffer )
-  expected_buffer = [ "aAa", "aDb", "Ea", "aCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "aDb" ),
+                      ToBytes( "Ea" ),
+                      ToBytes( "aCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 1 )
   eq_( char_offset, -1 )
 
 
 def ReplaceChunk_MultipleLinesToSingleLineOffsetWorks_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 1, 1, 2, 2 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb',
                                                           1, 1, result_buffer )
-  expected_buffer = [ "aAa", "abDbCa" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "abDbCa" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, -1 )
   eq_( char_offset, 3 )
 
 
 def ReplaceChunk_MultipleLineOffsetWorks_test():
-  result_buffer = [ "aAa", "aBa", "aCa" ]
+  result_buffer = [ ToBytes( "aAa" ),
+                    ToBytes( "aBa" ),
+                    ToBytes( "aCa" ) ]
   start, end = _BuildLocations( 3, 1, 4, 3 )
   ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
                                                           end,
@@ -534,7 +650,10 @@ def ReplaceChunk_MultipleLineOffsetWorks_test():
                                                           -1,
                                                           1,
                                                           result_buffer )
-  expected_buffer = [ "aAa", "abDb", "bEb", "bFba" ]
+  expected_buffer = [ ToBytes( "aAa" ),
+                      ToBytes( "abDb" ),
+                      ToBytes( "bEb" ),
+                      ToBytes( "bFba" ) ]
   eq_( expected_buffer, result_buffer )
   eq_( line_offset, 1 )
   eq_( char_offset, 1 )
@@ -556,10 +675,10 @@ def ReplaceChunksInBuffer_SortedChunks_test():
     _BuildChunk( 1, 11, 1, 11, ')' )
   ]
 
-  result_buffer = [ "CT<10 >> 2> ct" ]
+  result_buffer = [ ToBytes( "CT<10 >> 2> ct" ) ]
   vimsupport.ReplaceChunksInBuffer( chunks, result_buffer, None )
 
-  expected_buffer = [ "CT<(10 >> 2)> ct" ]
+  expected_buffer = [ ToBytes( "CT<(10 >> 2)> ct" ) ]
   eq_( expected_buffer, result_buffer )
 
 
@@ -569,10 +688,10 @@ def ReplaceChunksInBuffer_UnsortedChunks_test():
     _BuildChunk( 1, 4, 1, 4, '(' )
   ]
 
-  result_buffer = [ "CT<10 >> 2> ct" ]
+  result_buffer = [ ToBytes( "CT<10 >> 2> ct" ) ]
   vimsupport.ReplaceChunksInBuffer( chunks, result_buffer, None )
 
-  expected_buffer = [ "CT<(10 >> 2)> ct" ]
+  expected_buffer = [ ToBytes( "CT<(10 >> 2)> ct" ) ]
   eq_( expected_buffer, result_buffer )
 
 
@@ -581,12 +700,13 @@ class MockBuffer( object ):
   generate a location list"""
 
   def __init__( self, lines, name, number ):
-    self.lines = lines
+    self.lines = [ ToBytes( x ) for x in lines ]
     self.name = name
     self.number = number
 
 
   def __getitem__( self, index ):
+    """ Return the bytes for a given line at index |index| """
     return self.lines[ index ]
 
 
@@ -598,6 +718,11 @@ class MockBuffer( object ):
     return self.lines.__setitem__( key, value )
 
 
+  def GetLines( self ):
+    """ Return the contents of the buffer as a list of unicode strings"""
+    return [ ToUnicode( x ) for x in self.lines ]
+
+
 @patch( 'ycm.vimsupport.GetBufferNumberForFilename',
         return_value=1,
         new_callable=ExtendedMock )
@@ -629,7 +754,7 @@ def ReplaceChunks_SingleFile_Open_test( vim_command,
     vimsupport.ReplaceChunks( chunks )
 
   # Ensure that we applied the replacement correctly
-  eq_( result_buffer.lines, [
+  eq_( result_buffer.GetLines(), [
     'replacementline2',
     'line3',
   ] )
@@ -715,7 +840,7 @@ def ReplaceChunks_SingleFile_NotOpen_test( vim_command,
   ] )
 
   # Ensure that we applied the replacement correctly
-  eq_( result_buffer.lines, [
+  eq_( result_buffer.GetLines(), [
     'replacementline2',
     'line3',
   ] )
@@ -823,7 +948,7 @@ def ReplaceChunks_User_Declines_To_Open_File_test(
   ] )
 
   # Ensure that buffer is not changed
-  eq_( result_buffer.lines, [
+  eq_( result_buffer.GetLines(), [
     'line1',
     'line2',
     'line3',
@@ -905,7 +1030,7 @@ def ReplaceChunks_User_Aborts_Opening_File_test(
   ] )
 
   # Ensure that buffer is not changed
-  eq_( result_buffer.lines, [
+  eq_( result_buffer.GetLines(), [
     'line1',
     'line2',
     'line3',
@@ -996,11 +1121,11 @@ def ReplaceChunks_MultiFile_Open_test( vim_command,
   ] )
 
   # Ensure that buffers are updated
-  eq_( another_file.lines, [
+  eq_( another_file.GetLines(), [
     'another line1',
     'second_file_replacement ACME line2',
   ] )
-  eq_( first_file.lines, [
+  eq_( first_file.GetLines(), [
     'first_file_replacement line2',
     'line3',
   ] )

python/ycm/vimsupport.py

@@ -30,7 +30,7 @@ import tempfile
 import json
 import re
 from collections import defaultdict
-from ycmd.utils import ToUnicode
+from ycmd.utils import ToUnicode, ToBytes
 from ycmd import user_options_store
 
 BUFFER_COMMAND_MAP = { 'same-buffer'      : 'edit',
@@ -232,7 +232,7 @@ def AddDiagnosticSyntaxMatch( line_num,
 
 
 # Clamps the line and column numbers so that they are not past the contents of
-# the buffer. Numbers are 1-based.
+# the buffer. Numbers are 1-based byte offsets.
 def LineAndColumnNumbersClamped( line_num, column_num ):
   new_line_num = line_num
   new_column_num = column_num
@@ -690,6 +690,9 @@ def ReplaceChunksInBuffer( chunks, vim_buffer, locations ):
 #
 # returns the delta (in lines and characters) that any position after the end
 # needs to be adjusted by.
+#
+# NOTE: Works exclusively with bytes() instances and byte offsets as returned
+# by ycmd and used within the Vim buffers
 def ReplaceChunk( start, end, replacement_text, line_delta, char_delta,
                   vim_buffer, locations = None ):
   # ycmd's results are all 1-based, but vim's/python's are all 0-based
@@ -703,9 +706,11 @@ def ReplaceChunk( start, end, replacement_text, line_delta, char_delta,
   if source_lines_count == 1:
     end_column += char_delta
 
-  replacement_lines = replacement_text.splitlines( False )
+  # NOTE: replacement_text is unicode, but all our offsets are byte offsets,
+  # so we convert to bytes
+  replacement_lines = ToBytes( replacement_text ).splitlines( False )
   if not replacement_lines:
-    replacement_lines = [ '' ]
+    replacement_lines = [ bytes( b'' ) ]
   replacement_lines_count = len( replacement_lines )
 
   end_existing_text = vim_buffer[ end_line ][ end_column : ]

python/ycm/youcompleteme.py

@@ -400,7 +400,8 @@ class YouCompleteMe( object ):
         self._HasCompletionsThatCouldBeCompletedWithMoreText_OlderVim
 
 
-  def _FilterToMatchingCompletions_NewerVim( self, completions,
+  def _FilterToMatchingCompletions_NewerVim( self,
+                                             completions,
                                              full_match_only ):
     """Filter to completions matching the item Vim said was completed"""
     completed = vimsupport.GetVariableValue( 'v:completed_item' )
@@ -410,7 +411,8 @@ class YouCompleteMe( object ):
                       else [ 'word' ] )
 
       def matcher( key ):
-          return completed.get( key, "" ) == item.get( key, "" )
+        return ( utils.ToUnicode( completed.get( key, "" ) ) ==
+                 utils.ToUnicode( item.get( key, "" ) ) )
 
       if all( [ matcher( i ) for i in match_keys ] ):
         yield completion
@@ -438,12 +440,12 @@ class YouCompleteMe( object ):
     if not completed_item:
       return False
 
-    completed_word = completed_item[ 'word' ]
+    completed_word = utils.ToUnicode( completed_item[ 'word' ] )
     if not completed_word:
       return False
 
-    # Sometime CompleteDone is called after the next character is inserted
+    # Sometimes CompleteDone is called after the next character is inserted.
-    # If so, use inserted character to filter possible completions further
+    # If so, use inserted character to filter possible completions further.
     text = vimsupport.TextBeforeCursor()
     reject_exact_match = True
     if text and text[ -1 ] != completed_word[ -1 ]:
@@ -451,7 +453,8 @@ class YouCompleteMe( object ):
       completed_word += text[ -1 ]
 
     for completion in completions:
-      word = ConvertCompletionDataToVimData( completion )[ 'word' ]
+      word = utils.ToUnicode(
+          ConvertCompletionDataToVimData( completion )[ 'word' ] )
       if reject_exact_match and word == completed_word:
         continue
       if word.startswith( completed_word ):
@@ -464,14 +467,14 @@ class YouCompleteMe( object ):
     # No support for multiple line completions
     text = vimsupport.TextBeforeCursor()
     for completion in completions:
-      word = ConvertCompletionDataToVimData( completion )[ 'word' ]
+      word = utils.ToUnicode(
+          ConvertCompletionDataToVimData( completion )[ 'word' ] )
       for i in range( 1, len( word ) - 1 ): # Excluding full word
         if text[ -1 * i : ] == word[ : i ]:
           return True
     return False
 
 
-
   def _OnCompleteDone_Csharp( self ):
     completions = self.GetCompletionsUserMayHaveCompleted()
     namespaces = [ self._GetRequiredNamespaceImport( c )

run_tests.py

@@ -47,6 +47,8 @@ def ParseArguments():
   parser = argparse.ArgumentParser()
   parser.add_argument( '--skip-build', action = 'store_true',
                        help = 'Do not build ycmd before testing.' )
+  parser.add_argument( '--no-flake8', action = 'store_true',
+                       help = 'Do not run flake8' )
 
   return parser.parse_known_args()
 
@@ -70,6 +72,7 @@ def NoseTests( extra_args ):
 
 def Main():
   ( parsed_args, extra_args ) = ParseArguments()
+  if not parsed_args.no_flake8:
     RunFlake8()
   BuildYcmdLibs( parsed_args )
   NoseTests( extra_args )