From 4d7b386a374490c0085eee937c12c80f1d108dae Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sat, 26 Mar 2016 03:40:17 +0000 Subject: [PATCH] Fix a number of multi-byte errors and tracebacks - Correct FixIts when there are unicode characters - Fix errors in CompleteDone handler - Fix tracebacks when omnifunc returns unicode chars --- python/ycm/omni_completer.py | 7 +- python/ycm/tests/postcomplete_tests.py | 85 +++++-- python/ycm/tests/vimsupport_test.py | 309 +++++++++++++++++-------- python/ycm/vimsupport.py | 13 +- python/ycm/youcompleteme.py | 23 +- 5 files changed, 308 insertions(+), 129 deletions(-) diff --git a/python/ycm/omni_completer.py b/python/ycm/omni_completer.py index 7e3b77db..b3be6f66 100644 --- a/python/ycm/omni_completer.py +++ b/python/ycm/omni_completer.py @@ -25,6 +25,7 @@ from builtins import * # noqa import vim from ycm import vimsupport +from ycmd import utils from ycmd.responses import ServerError from ycmd.completers.completer import Completer from ycm.client.base_request import BaseRequest, HandleServerException @@ -89,12 +90,14 @@ class OmniCompleter( Completer ): items = vim.eval( ''.join( omnifunc_call ) ) - if 'words' in items: + if hasattr( items, '__getitem__' ) and 'words' in items: items = items[ 'words' ] + if not hasattr( items, '__iter__' ): raise TypeError( OMNIFUNC_NOT_LIST ) - return list( filter( bool, items ) ) + return [ utils.ToUnicode( i ) for i in items if bool( i ) ] + except ( TypeError, ValueError, vim.error ) as error: vimsupport.PostVimMessage( OMNIFUNC_RETURNED_BAD_VALUE + ' ' + str( error ) ) diff --git a/python/ycm/tests/postcomplete_tests.py b/python/ycm/tests/postcomplete_tests.py index fbcacd88..02c03379 100644 --- a/python/ycm/tests/postcomplete_tests.py +++ b/python/ycm/tests/postcomplete_tests.py @@ -1,3 +1,5 @@ +# encoding: utf-8 +# # Copyright (C) 2015 YouCompleteMe contributors # # This file is part of YouCompleteMe. @@ -26,6 +28,8 @@ from builtins import * # noqa from ycm.test_utils import MockVimModule MockVimModule() +from ycmd.utils import ToBytes + import contextlib from hamcrest import assert_that, empty from mock import MagicMock, DEFAULT, patch @@ -40,11 +44,11 @@ def GetVariableValue_CompleteItemIs( word, abbr = None, menu = None, def Result( variable ): if variable == 'v:completed_item': return { - 'word': word, - 'abbr': abbr, - 'menu': menu, - 'info': info, - 'kind': kind, + 'word': ToBytes( word ), + 'abbr': ToBytes( abbr ), + 'menu': ToBytes( menu ), + 'info': ToBytes( info ), + 'kind': ToBytes( kind ), } return DEFAULT return MagicMock( side_effect = Result ) @@ -115,7 +119,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.GetVariableValue', GetVariableValue_CompleteItemIs( 'Test' ) ) def FilterToCompletedCompletions_NewVim_MatchIsReturned_test( self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._FilterToMatchingCompletions( completions, False ) eq_( list( result ), completions ) @@ -125,7 +129,7 @@ class PostComplete_test(): GetVariableValue_CompleteItemIs( 'A' ) ) def FilterToCompletedCompletions_NewVim_ShortTextDoesntRaise_test( self, *args ): - completions = [ BuildCompletion( 'AAA' ) ] + completions = [ BuildCompletion( insertion_text = 'AAA' ) ] self.ycm._FilterToMatchingCompletions( completions, False ) @@ -134,7 +138,7 @@ class PostComplete_test(): GetVariableValue_CompleteItemIs( 'Test' ) ) def FilterToCompletedCompletions_NewVim_ExactMatchIsReturned_test( self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._FilterToMatchingCompletions( completions, False ) eq_( list( result ), completions ) @@ -144,15 +148,24 @@ class PostComplete_test(): GetVariableValue_CompleteItemIs( ' Quote' ) ) def FilterToCompletedCompletions_NewVim_NonMatchIsntReturned_test( self, *args ): - completions = [ BuildCompletion( 'A' ) ] + completions = [ BuildCompletion( insertion_text = 'A' ) ] result = self.ycm._FilterToMatchingCompletions( completions, False ) assert_that( list( result ), empty() ) + @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = True ) + @patch( 'ycm.vimsupport.GetVariableValue', + GetVariableValue_CompleteItemIs( '†es†' ) ) + def FilterToCompletedCompletions_NewVim_Unicode_test( self, *args ): + completions = [ BuildCompletion( insertion_text = '†es†' ) ] + result = self.ycm._FilterToMatchingCompletions( completions, False ) + eq_( list( result ), completions ) + + @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = False ) @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Test' ) def FilterToCompletedCompletions_OldVim_MatchIsReturned_test( self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._FilterToMatchingCompletions( completions, False ) eq_( list( result ), completions ) @@ -161,7 +174,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'X' ) def FilterToCompletedCompletions_OldVim_ShortTextDoesntRaise_test( self, *args ): - completions = [ BuildCompletion( 'AAA' ) ] + completions = [ BuildCompletion( insertion_text = 'AAA' ) ] self.ycm._FilterToMatchingCompletions( completions, False ) @@ -169,7 +182,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Test' ) def FilterToCompletedCompletions_OldVim_ExactMatchIsReturned_test( self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._FilterToMatchingCompletions( completions, False ) eq_( list( result ), completions ) @@ -178,7 +191,15 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Quote' ) def FilterToCompletedCompletions_OldVim_NonMatchIsntReturned_test( self, *args ): - completions = [ BuildCompletion( 'A' ) ] + completions = [ BuildCompletion( insertion_text = 'A' ) ] + result = self.ycm._FilterToMatchingCompletions( completions, False ) + assert_that( list( result ), empty() ) + + + @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = False ) + @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Uniçø∂¢' ) + def FilterToCompletedCompletions_OldVim_Unicode_test( self, *args ): + completions = [ BuildCompletion( insertion_text = 'Uniçø∂¢' ) ] result = self.ycm._FilterToMatchingCompletions( completions, False ) assert_that( list( result ), empty() ) @@ -187,7 +208,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Te' ) def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_MatchIsReturned_test( # noqa self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) eq_( result, True ) @@ -197,7 +218,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'X' ) def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_ShortTextDoesntRaise_test( # noqa self, *args ): - completions = [ BuildCompletion( "AAA" ) ] + completions = [ BuildCompletion( insertion_text = "AAA" ) ] self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) @@ -205,7 +226,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Test' ) def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_ExactMatchIsntReturned_test( # noqa self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) eq_( result, False ) @@ -215,19 +236,29 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Quote' ) def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_NonMatchIsntReturned_test( # noqa self, *args ): - completions = [ BuildCompletion( 'A' ) ] + completions = [ BuildCompletion( insertion_text = 'A' ) ] result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) eq_( result, False ) + @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = False ) + @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Uniç' ) + def HasCompletionsThatCouldBeCompletedWithMoreText_OldVim_Unicode_test( + self, *args ): + completions = [ BuildCompletion( insertion_text = 'Uniçø∂¢' ) ] + result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( + completions ) + eq_( result, True ) + + @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = True ) @patch( 'ycm.vimsupport.GetVariableValue', GetVariableValue_CompleteItemIs( 'Te' ) ) @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Quote' ) def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_MatchIsReturned_test( # noqa self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) eq_( result, True ) @@ -239,7 +270,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Quote' ) def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_ShortTextDoesntRaise_test( # noqa self, *args ): - completions = [ BuildCompletion( 'AAA' ) ] + completions = [ BuildCompletion( insertion_text = 'AAA' ) ] self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) @@ -249,7 +280,7 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Quote' ) def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_ExactMatchIsntReturned_test( # noqa self, *args ): - completions = [ BuildCompletion( 'Test' ) ] + completions = [ BuildCompletion( insertion_text = 'Test' ) ] result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) eq_( result, False ) @@ -261,12 +292,24 @@ class PostComplete_test(): @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = ' Quote' ) def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_NonMatchIsntReturned_test( # noqa self, *args ): - completions = [ BuildCompletion( "A" ) ] + completions = [ BuildCompletion( insertion_text = "A" ) ] result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( completions ) eq_( result, False ) + @patch( 'ycm.vimsupport.VimVersionAtLeast', return_value = True ) + @patch( 'ycm.vimsupport.GetVariableValue', + GetVariableValue_CompleteItemIs( 'Uniç' ) ) + @patch( 'ycm.vimsupport.TextBeforeCursor', return_value = 'Uniç' ) + def HasCompletionsThatCouldBeCompletedWithMoreText_NewVim_Unicode_test( + self, *args ): + completions = [ BuildCompletion( insertion_text = "Uniçø∂¢" ) ] + result = self.ycm._HasCompletionsThatCouldBeCompletedWithMoreText( + completions ) + eq_( result, True ) + + def GetRequiredNamespaceImport_ReturnNoneForNoExtraData_test( self ): eq_( None, self.ycm._GetRequiredNamespaceImport( {} ) ) diff --git a/python/ycm/tests/vimsupport_test.py b/python/ycm/tests/vimsupport_test.py index b8137efb..bb0e7204 100644 --- a/python/ycm/tests/vimsupport_test.py +++ b/python/ycm/tests/vimsupport_test.py @@ -32,15 +32,14 @@ from ycm import vimsupport from nose.tools import eq_ from hamcrest import assert_that, calling, raises, none, has_entry from mock import MagicMock, call, patch -from ycmd.utils import ToBytes +from ycmd.utils import ToBytes, ToUnicode import os import json def ReplaceChunk_SingleLine_Repl_1_test(): # Replace with longer range - # 12345678901234567 - result_buffer = [ "This is a string" ] + result_buffer = [ ToBytes( "This is a string" ) ] start, end = _BuildLocations( 1, 1, 1, 5 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -49,7 +48,7 @@ def ReplaceChunk_SingleLine_Repl_1_test(): 0, result_buffer ) - eq_( [ "How long is a string" ], result_buffer ) + eq_( [ ToBytes( "How long is a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 4 ) @@ -66,7 +65,7 @@ def ReplaceChunk_SingleLine_Repl_1_test(): line_offset += new_line_offset char_offset += new_char_offset - eq_( [ 'How long is a piece of string' ], result_buffer ) + eq_( [ ToBytes( 'How long is a piece of string' ) ], result_buffer ) eq_( new_line_offset, 0 ) eq_( new_char_offset, 9 ) eq_( line_offset, 0 ) @@ -86,7 +85,7 @@ def ReplaceChunk_SingleLine_Repl_1_test(): line_offset += new_line_offset char_offset += new_char_offset - eq_( ['How long is a piece of pie' ], result_buffer ) + eq_( [ ToBytes( 'How long is a piece of pie' ) ], result_buffer ) eq_( new_line_offset, 0 ) eq_( new_char_offset, -3 ) eq_( line_offset, 0 ) @@ -95,8 +94,7 @@ def ReplaceChunk_SingleLine_Repl_1_test(): def ReplaceChunk_SingleLine_Repl_2_test(): # Replace with shorter range - # 12345678901234567 - result_buffer = [ "This is a string" ] + result_buffer = [ ToBytes( "This is a string" ) ] start, end = _BuildLocations( 1, 11, 1, 17 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -105,15 +103,14 @@ def ReplaceChunk_SingleLine_Repl_2_test(): 0, result_buffer ) - eq_( [ "This is a test" ], result_buffer ) + eq_( [ ToBytes( "This is a test" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, -2 ) def ReplaceChunk_SingleLine_Repl_3_test(): # Replace with equal range - # 12345678901234567 - result_buffer = [ "This is a string" ] + result_buffer = [ ToBytes( "This is a string" ) ] start, end = _BuildLocations( 1, 6, 1, 8 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -122,14 +119,14 @@ def ReplaceChunk_SingleLine_Repl_3_test(): 0, result_buffer ) - eq_( [ "This be a string" ], result_buffer ) + eq_( [ ToBytes( "This be a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 0 ) def ReplaceChunk_SingleLine_Add_1_test(): # Insert at start - result_buffer = [ "is a string" ] + result_buffer = [ ToBytes( "is a string" ) ] start, end = _BuildLocations( 1, 1, 1, 1 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -138,14 +135,14 @@ def ReplaceChunk_SingleLine_Add_1_test(): 0, result_buffer ) - eq_( [ "This is a string" ], result_buffer ) + eq_( [ ToBytes( "This is a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 5 ) def ReplaceChunk_SingleLine_Add_2_test(): # Insert at end - result_buffer = [ "This is a " ] + result_buffer = [ ToBytes( "This is a " ) ] start, end = _BuildLocations( 1, 11, 1, 11 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -154,14 +151,14 @@ def ReplaceChunk_SingleLine_Add_2_test(): 0, result_buffer ) - eq_( [ "This is a string" ], result_buffer ) + eq_( [ ToBytes( "This is a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 6 ) def ReplaceChunk_SingleLine_Add_3_test(): # Insert in the middle - result_buffer = [ "This is a string" ] + result_buffer = [ ToBytes( "This is a string" ) ] start, end = _BuildLocations( 1, 8, 1, 8 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -170,14 +167,14 @@ def ReplaceChunk_SingleLine_Add_3_test(): 0, result_buffer ) - eq_( [ "This is not a string" ], result_buffer ) + eq_( [ ToBytes( "This is not a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 4 ) def ReplaceChunk_SingleLine_Del_1_test(): # Delete from start - result_buffer = [ "This is a string" ] + result_buffer = [ ToBytes( "This is a string" ) ] start, end = _BuildLocations( 1, 1, 1, 6 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -186,14 +183,14 @@ def ReplaceChunk_SingleLine_Del_1_test(): 0, result_buffer ) - eq_( [ "is a string" ], result_buffer ) + eq_( [ ToBytes( "is a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, -5 ) def ReplaceChunk_SingleLine_Del_2_test(): # Delete from end - result_buffer = [ "This is a string" ] + result_buffer = [ ToBytes( "This is a string" ) ] start, end = _BuildLocations( 1, 10, 1, 18 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -202,14 +199,14 @@ def ReplaceChunk_SingleLine_Del_2_test(): 0, result_buffer ) - eq_( [ "This is a" ], result_buffer ) + eq_( [ ToBytes( "This is a" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, -8 ) def ReplaceChunk_SingleLine_Del_3_test(): # Delete from middle - result_buffer = [ "This is not a string" ] + result_buffer = [ ToBytes( "This is not a string" ) ] start, end = _BuildLocations( 1, 9, 1, 13 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -218,33 +215,84 @@ def ReplaceChunk_SingleLine_Del_3_test(): 0, result_buffer ) - eq_( [ "This is a string" ], result_buffer ) + eq_( [ ToBytes( "This is a string" ) ], result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, -4 ) +def ReplaceChunk_SingleLine_Unicode_ReplaceUnicodeChars_test(): + # Replace Unicode characters. + result_buffer = [ ToBytes( "This Uniçø∂‰ string is in the middle" ) ] + start, end = _BuildLocations( 1, 6, 1, 20 ) + ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, + end, + 'Unicode ', + 0, + 0, + result_buffer ) + + eq_( [ ToBytes( "This Unicode string is in the middle" ) ], result_buffer ) + eq_( line_offset, 0 ) + eq_( char_offset, -6 ) + + +def ReplaceChunk_SingleLine_Unicode_ReplaceAfterUnicode_test(): + # Replace ASCII characters after Unicode characters in the line. + result_buffer = [ ToBytes( "This Uniçø∂‰ string is in the middle" ) ] + start, end = _BuildLocations( 1, 30, 1, 43 ) + ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, + end, + 'fåke', + 0, + 0, + result_buffer ) + + eq_( [ ToBytes( "This Uniçø∂‰ string is fåke" ) ], result_buffer ) + eq_( line_offset, 0 ) + eq_( char_offset, -8 ) + + +def ReplaceChunk_SingleLine_Unicode_Grown_test(): + # Replace ASCII characters after Unicode characters in the line. + result_buffer = [ ToBytes( "a" ) ] + start, end = _BuildLocations( 1, 1, 1, 2 ) + ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, + end, + 'å', + 0, + 0, + result_buffer ) + + eq_( [ ToBytes( "å" ) ], result_buffer ) + eq_( line_offset, 0 ) + eq_( char_offset, 1 ) # Note: byte difference (a = 1 byte, å = 2 bytes) + + def ReplaceChunk_RemoveSingleLine_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 1, 3, 1 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, '', 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, -1 ) eq_( char_offset, 0 ) def ReplaceChunk_SingleToMultipleLines_test(): - result_buffer = [ "aAa", - "aBa", - "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 2, 2, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF', 0, 0, result_buffer ) - expected_buffer = [ "aAa", - "aEb", - "bFBa", - "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aEb" ), + ToBytes( "bFBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 1 ) eq_( char_offset, 1 ) @@ -262,13 +310,16 @@ def ReplaceChunk_SingleToMultipleLines_test(): line_offset += new_line_offset char_offset += new_char_offset - eq_( [ "aAa", "aEb", "bFBcccc", "aCa" ], result_buffer ) + eq_( [ ToBytes( "aAa" ), + ToBytes( "aEb" ), + ToBytes( "bFBcccc" ), + ToBytes( "aCa" ) ], result_buffer ) eq_( line_offset, 1 ) eq_( char_offset, 4 ) def ReplaceChunk_SingleToMultipleLines2_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 2, 2, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -276,14 +327,18 @@ def ReplaceChunk_SingleToMultipleLines2_test(): 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aEb", "bFb", "GBa", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aEb" ), + ToBytes( "bFb" ), + ToBytes( "GBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 2 ) eq_( char_offset, 0 ) def ReplaceChunk_SingleToMultipleLines3_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 2, 2, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -291,14 +346,18 @@ def ReplaceChunk_SingleToMultipleLines3_test(): 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aEb", "bFb", "bGbBa", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aEb" ), + ToBytes( "bFb" ), + ToBytes( "bGbBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 2 ) eq_( char_offset, 2 ) def ReplaceChunk_SingleToMultipleLinesReplace_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ] start, end = _BuildLocations( 1, 2, 1, 4 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -306,16 +365,20 @@ def ReplaceChunk_SingleToMultipleLinesReplace_test(): 0, 0, result_buffer ) - expected_buffer = [ "aEb", "bFb", "bGb", "aBa", "aCa" ] + expected_buffer = [ ToBytes( "aEb" ), + ToBytes( "bFb" ), + ToBytes( "bGb" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 2 ) eq_( char_offset, 0 ) def ReplaceChunk_SingleToMultipleLinesReplace_2_test(): - result_buffer = [ "aAa", - "aBa", - "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 1, 2, 1, 4 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -323,11 +386,11 @@ def ReplaceChunk_SingleToMultipleLinesReplace_2_test(): 0, 0, result_buffer ) - expected_buffer = [ "aEb", - "bFb", - "bGb", - "aBa", - "aCa" ] + expected_buffer = [ ToBytes( "aEb" ), + ToBytes( "bFb" ), + ToBytes( "bGb" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 2 ) eq_( char_offset, 0 ) @@ -345,22 +408,22 @@ def ReplaceChunk_SingleToMultipleLinesReplace_2_test(): line_offset += new_line_offset char_offset += new_char_offset - eq_( [ "aEb", - "bFb", - "bGbcccc", - "aBa", - "aCa" ], result_buffer ) + eq_( [ ToBytes( "aEb" ), + ToBytes( "bFb" ), + ToBytes( "bGbcccc" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ], result_buffer ) eq_( line_offset, 2 ) eq_( char_offset, 4 ) def ReplaceChunk_MultipleLinesToSingleLine_test(): - result_buffer = [ "aAa", "aBa", "aCaaaa" ] + result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCaaaa" ) ] start, end = _BuildLocations( 2, 2, 3, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'E', 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aECaaaa" ] + expected_buffer = [ ToBytes( "aAa" ), ToBytes( "aECaaaa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, -1 ) eq_( char_offset, 1 ) @@ -378,7 +441,8 @@ def ReplaceChunk_MultipleLinesToSingleLine_test(): line_offset += new_line_offset char_offset += new_char_offset - eq_( [ "aAa", "aECccccaaa" ], result_buffer ) + eq_( [ ToBytes( "aAa" ), + ToBytes( "aECccccaaa" ) ], result_buffer ) eq_( line_offset, -1 ) eq_( char_offset, 4 ) @@ -395,24 +459,36 @@ def ReplaceChunk_MultipleLinesToSingleLine_test(): line_offset += new_line_offset char_offset += new_char_offset - eq_( [ "aAa", "aECccccdd", "ddaa" ], result_buffer ) + eq_( [ ToBytes( "aAa" ), + ToBytes( "aECccccdd" ), + ToBytes( "ddaa" ) ], + result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, -2 ) def ReplaceChunk_MultipleLinesToSameMultipleLines_test(): - result_buffer = [ "aAa", "aBa", "aCa", "aDe" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ), + ToBytes( "aDe" ) ] start, end = _BuildLocations( 2, 2, 3, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF', 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aEb", "bFCa", "aDe" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aEb" ), + ToBytes( "bFCa" ), + ToBytes( "aDe" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 1 ) def ReplaceChunk_MultipleLinesToMoreMultipleLines_test(): - result_buffer = [ "aAa", "aBa", "aCa", "aDe" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ), + ToBytes( "aDe" ) ] start, end = _BuildLocations( 2, 2, 3, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -420,113 +496,153 @@ def ReplaceChunk_MultipleLinesToMoreMultipleLines_test(): 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aEb", "bFb", "bGCa", "aDe" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aEb" ), + ToBytes( "bFb" ), + ToBytes( "bGCa" ), + ToBytes( "aDe" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 1 ) eq_( char_offset, 1 ) def ReplaceChunk_MultipleLinesToLessMultipleLines_test(): - result_buffer = [ "aAa", "aBa", "aCa", "aDe" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ), + ToBytes( "aDe" ) ] start, end = _BuildLocations( 1, 2, 3, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF', 0, 0, result_buffer ) - expected_buffer = [ "aEb", "bFCa", "aDe" ] + expected_buffer = [ ToBytes( "aEb" ), ToBytes( "bFCa" ), ToBytes( "aDe" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, -1 ) eq_( char_offset, 1 ) def ReplaceChunk_MultipleLinesToEvenLessMultipleLines_test(): - result_buffer = [ "aAa", "aBa", "aCa", "aDe" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ), + ToBytes( "aDe" ) ] start, end = _BuildLocations( 1, 2, 4, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF', 0, 0, result_buffer ) - expected_buffer = [ "aEb", "bFDe" ] + expected_buffer = [ ToBytes( "aEb" ), ToBytes( "bFDe" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, -2 ) eq_( char_offset, 1 ) def ReplaceChunk_SpanBufferEdge_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 1, 1, 1, 3 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb', 0, 0, result_buffer ) - expected_buffer = [ "bDba", "aBa", "aCa" ] + expected_buffer = [ ToBytes( "bDba" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 1 ) def ReplaceChunk_DeleteTextInLine_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 2, 2, 3 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, '', 0, 0, result_buffer ) - expected_buffer = [ "aAa", "aa", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, -1 ) def ReplaceChunk_AddTextInLine_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 2, 2, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb', 0, 0, result_buffer ) - expected_buffer = [ "aAa", "abDbBa", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "abDbBa" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 3 ) def ReplaceChunk_ReplaceTextInLine_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 2, 2, 2, 3 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb', 0, 0, result_buffer ) - expected_buffer = [ "aAa", "abDba", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "abDba" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 2 ) def ReplaceChunk_SingleLineOffsetWorks_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 1, 1, 1, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb', 1, 1, result_buffer ) - expected_buffer = [ "aAa", "abDba", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "abDba" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 0 ) eq_( char_offset, 2 ) def ReplaceChunk_SingleLineToMultipleLinesOffsetWorks_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 1, 1, 1, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Db\nE', 1, 1, result_buffer ) - expected_buffer = [ "aAa", "aDb", "Ea", "aCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "aDb" ), + ToBytes( "Ea" ), + ToBytes( "aCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 1 ) eq_( char_offset, -1 ) def ReplaceChunk_MultipleLinesToSingleLineOffsetWorks_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 1, 1, 2, 2 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'bDb', 1, 1, result_buffer ) - expected_buffer = [ "aAa", "abDbCa" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "abDbCa" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, -1 ) eq_( char_offset, 3 ) def ReplaceChunk_MultipleLineOffsetWorks_test(): - result_buffer = [ "aAa", "aBa", "aCa" ] + result_buffer = [ ToBytes( "aAa" ), + ToBytes( "aBa" ), + ToBytes( "aCa" ) ] start, end = _BuildLocations( 3, 1, 4, 3 ) ( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, @@ -534,7 +650,10 @@ def ReplaceChunk_MultipleLineOffsetWorks_test(): -1, 1, result_buffer ) - expected_buffer = [ "aAa", "abDb", "bEb", "bFba" ] + expected_buffer = [ ToBytes( "aAa" ), + ToBytes( "abDb" ), + ToBytes( "bEb" ), + ToBytes( "bFba" ) ] eq_( expected_buffer, result_buffer ) eq_( line_offset, 1 ) eq_( char_offset, 1 ) @@ -556,10 +675,10 @@ def ReplaceChunksInBuffer_SortedChunks_test(): _BuildChunk( 1, 11, 1, 11, ')' ) ] - result_buffer = [ "CT<10 >> 2> ct" ] + result_buffer = [ ToBytes( "CT<10 >> 2> ct" ) ] vimsupport.ReplaceChunksInBuffer( chunks, result_buffer, None ) - expected_buffer = [ "CT<(10 >> 2)> ct" ] + expected_buffer = [ ToBytes( "CT<(10 >> 2)> ct" ) ] eq_( expected_buffer, result_buffer ) @@ -569,10 +688,10 @@ def ReplaceChunksInBuffer_UnsortedChunks_test(): _BuildChunk( 1, 4, 1, 4, '(' ) ] - result_buffer = [ "CT<10 >> 2> ct" ] + result_buffer = [ ToBytes( "CT<10 >> 2> ct" ) ] vimsupport.ReplaceChunksInBuffer( chunks, result_buffer, None ) - expected_buffer = [ "CT<(10 >> 2)> ct" ] + expected_buffer = [ ToBytes( "CT<(10 >> 2)> ct" ) ] eq_( expected_buffer, result_buffer ) @@ -581,12 +700,13 @@ class MockBuffer( object ): generate a location list""" def __init__( self, lines, name, number ): - self.lines = lines + self.lines = [ ToBytes( x ) for x in lines ] self.name = name self.number = number def __getitem__( self, index ): + """ Return the bytes for a given line at index |index| """ return self.lines[ index ] @@ -598,6 +718,11 @@ class MockBuffer( object ): return self.lines.__setitem__( key, value ) + def GetLines( self ): + """ Return the contents of the buffer as a list of unicode strings""" + return [ ToUnicode( x ) for x in self.lines ] + + @patch( 'ycm.vimsupport.GetBufferNumberForFilename', return_value=1, new_callable=ExtendedMock ) @@ -629,7 +754,7 @@ def ReplaceChunks_SingleFile_Open_test( vim_command, vimsupport.ReplaceChunks( chunks ) # Ensure that we applied the replacement correctly - eq_( result_buffer.lines, [ + eq_( result_buffer.GetLines(), [ 'replacementline2', 'line3', ] ) @@ -715,7 +840,7 @@ def ReplaceChunks_SingleFile_NotOpen_test( vim_command, ] ) # Ensure that we applied the replacement correctly - eq_( result_buffer.lines, [ + eq_( result_buffer.GetLines(), [ 'replacementline2', 'line3', ] ) @@ -823,7 +948,7 @@ def ReplaceChunks_User_Declines_To_Open_File_test( ] ) # Ensure that buffer is not changed - eq_( result_buffer.lines, [ + eq_( result_buffer.GetLines(), [ 'line1', 'line2', 'line3', @@ -905,7 +1030,7 @@ def ReplaceChunks_User_Aborts_Opening_File_test( ] ) # Ensure that buffer is not changed - eq_( result_buffer.lines, [ + eq_( result_buffer.GetLines(), [ 'line1', 'line2', 'line3', @@ -996,11 +1121,11 @@ def ReplaceChunks_MultiFile_Open_test( vim_command, ] ) # Ensure that buffers are updated - eq_( another_file.lines, [ + eq_( another_file.GetLines(), [ 'another line1', 'second_file_replacement ACME line2', ] ) - eq_( first_file.lines, [ + eq_( first_file.GetLines(), [ 'first_file_replacement line2', 'line3', ] ) diff --git a/python/ycm/vimsupport.py b/python/ycm/vimsupport.py index 5c2302bc..b44ca109 100644 --- a/python/ycm/vimsupport.py +++ b/python/ycm/vimsupport.py @@ -30,7 +30,7 @@ import tempfile import json import re from collections import defaultdict -from ycmd.utils import ToUnicode +from ycmd.utils import ToUnicode, ToBytes from ycmd import user_options_store BUFFER_COMMAND_MAP = { 'same-buffer' : 'edit', @@ -232,7 +232,7 @@ def AddDiagnosticSyntaxMatch( line_num, # Clamps the line and column numbers so that they are not past the contents of -# the buffer. Numbers are 1-based. +# the buffer. Numbers are 1-based byte offsets. def LineAndColumnNumbersClamped( line_num, column_num ): new_line_num = line_num new_column_num = column_num @@ -690,6 +690,9 @@ def ReplaceChunksInBuffer( chunks, vim_buffer, locations ): # # returns the delta (in lines and characters) that any position after the end # needs to be adjusted by. +# +# NOTE: Works exclusively with bytes() instances and byte offsets as returned +# by ycmd and used within the Vim buffers def ReplaceChunk( start, end, replacement_text, line_delta, char_delta, vim_buffer, locations = None ): # ycmd's results are all 1-based, but vim's/python's are all 0-based @@ -703,9 +706,11 @@ def ReplaceChunk( start, end, replacement_text, line_delta, char_delta, if source_lines_count == 1: end_column += char_delta - replacement_lines = replacement_text.splitlines( False ) + # NOTE: replacement_text is unicode, but all our offsets are byte offsets, + # so we convert to bytes + replacement_lines = ToBytes( replacement_text ).splitlines( False ) if not replacement_lines: - replacement_lines = [ '' ] + replacement_lines = [ bytes( b'' ) ] replacement_lines_count = len( replacement_lines ) end_existing_text = vim_buffer[ end_line ][ end_column : ] diff --git a/python/ycm/youcompleteme.py b/python/ycm/youcompleteme.py index e3525b87..fa4bcf1f 100644 --- a/python/ycm/youcompleteme.py +++ b/python/ycm/youcompleteme.py @@ -400,9 +400,10 @@ class YouCompleteMe( object ): self._HasCompletionsThatCouldBeCompletedWithMoreText_OlderVim - def _FilterToMatchingCompletions_NewerVim( self, completions, + def _FilterToMatchingCompletions_NewerVim( self, + completions, full_match_only ): - """ Filter to completions matching the item Vim said was completed """ + """Filter to completions matching the item Vim said was completed""" completed = vimsupport.GetVariableValue( 'v:completed_item' ) for completion in completions: item = ConvertCompletionDataToVimData( completion ) @@ -410,7 +411,8 @@ class YouCompleteMe( object ): else [ 'word' ] ) def matcher( key ): - return completed.get( key, "" ) == item.get( key, "" ) + return ( utils.ToUnicode( completed.get( key, "" ) ) == + utils.ToUnicode( item.get( key, "" ) ) ) if all( [ matcher( i ) for i in match_keys ] ): yield completion @@ -438,12 +440,12 @@ class YouCompleteMe( object ): if not completed_item: return False - completed_word = completed_item[ 'word' ] + completed_word = utils.ToUnicode( completed_item[ 'word' ] ) if not completed_word: return False - # Sometime CompleteDone is called after the next character is inserted - # If so, use inserted character to filter possible completions further + # Sometimes CompleteDone is called after the next character is inserted. + # If so, use inserted character to filter possible completions further. text = vimsupport.TextBeforeCursor() reject_exact_match = True if text and text[ -1 ] != completed_word[ -1 ]: @@ -451,7 +453,8 @@ class YouCompleteMe( object ): completed_word += text[ -1 ] for completion in completions: - word = ConvertCompletionDataToVimData( completion )[ 'word' ] + word = utils.ToUnicode( + ConvertCompletionDataToVimData( completion )[ 'word' ] ) if reject_exact_match and word == completed_word: continue if word.startswith( completed_word ): @@ -464,14 +467,14 @@ class YouCompleteMe( object ): # No support for multiple line completions text = vimsupport.TextBeforeCursor() for completion in completions: - word = ConvertCompletionDataToVimData( completion )[ 'word' ] + word = utils.ToUnicode( + ConvertCompletionDataToVimData( completion )[ 'word' ] ) for i in range( 1, len( word ) - 1 ): # Excluding full word - if text[ -1 * i : ] == word[ : i ]: + if text[ -1 * i : ] == word[ : i ]: return True return False - def _OnCompleteDone_Csharp( self ): completions = self.GetCompletionsUserMayHaveCompleted() namespaces = [ self._GetRequiredNamespaceImport( c )