Auto merge of #2702 - micbou:unicode-warnings, r=bstaletic

[READY] Fix unicode warning when jumping on Python 2

On Python 2, jumping in a file whose path contains non-ASCII characters raises the following warnings:
```
Error detected while processing function <SNR>26_CompleterCommand:
line   18:
python\ycm\vimsupport.py:400: UnicodeWarning: Unicode unequal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
  if filename != GetCurrentBufferFilepath():
python\ycm\vimsupport.py:375: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
  if win.buffer.name == filepath:
```
This happens because, in the `JumpToLocation` function, we are comparing a unicode object (`filename` and `filepath`) with a Vim buffer name (`vim.current.buffer.name` returned by `GetCurrentBufferFilepath` and `win.buffer.name` where `win` is a Vim window) which is a byte object on Python 2.

For now, this PR adds tests covering the `JumpToLocation` function with unicode paths. They will raise the warnings on Python 2 and will fail because warnings are now treated as errors in tests. I'll update the PR with the fix once the builds are done.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/youcompleteme/2702)
<!-- Reviewable:end -->

python/ycm/tests/__init__.py

@@ -29,11 +29,12 @@ import functools
 import os
 import requests
 import time
+import warnings
 
 from ycm.client.base_request import BaseRequest
 from ycm.youcompleteme import YouCompleteMe
 from ycmd import user_options_store
-from ycmd.utils import WaitUntilProcessIsTerminated
+from ycmd.utils import CloseStandardStreams, WaitUntilProcessIsTerminated
 
 # The default options which are only relevant to the client, not the server and
 # thus are not part of default_options.json, but are required for a working
@@ -84,10 +85,23 @@ def StopServer( ycm ):
   try:
     ycm.OnVimLeave()
     WaitUntilProcessIsTerminated( ycm._server_popen )
+    CloseStandardStreams( ycm._server_popen )
   except Exception:
     pass
 
 
+def setUpPackage():
+  # We treat warnings as errors in our tests because warnings raised inside Vim
+  # will interrupt user workflow with a traceback and we don't want that.
+  warnings.filterwarnings( 'error' )
+  # We ignore warnings from nose as we are not interested in them.
+  warnings.filterwarnings( 'ignore', module = 'nose' )
+
+
+def tearDownPackage():
+  warnings.resetwarnings()
+
+
 def YouCompleteMeInstance( custom_options = {} ):
   """Defines a decorator function for tests that passes a unique YouCompleteMe
   instance as a parameter. This instance is initialized with the default options

python/ycm/tests/test_utils.py

@@ -91,6 +91,8 @@ def _MockGetBufferVariable( buffer_number, option ):
         return vim_buffer.filetype
       if option == 'changedtick':
         return vim_buffer.changedtick
+      if option == '&bh':
+        return vim_buffer.bufhidden
       return ''
   return ''
 
@@ -134,6 +136,9 @@ def _MockVimOptionsEval( value ):
   if value == '&showcmd':
     return 1
 
+  if value == '&hidden':
+    return 0
+
   return None
 
 
@@ -209,19 +214,21 @@ def MockVimCommand( command ):
 
 class VimBuffer( object ):
   """An object that looks like a vim.buffer object:
-   - |name|    : full path of the buffer with symbolic links resolved;
+   - |name|     : full path of the buffer with symbolic links resolved;
-   - |number|  : buffer number;
+   - |number|   : buffer number;
-   - |contents|: list of lines representing the buffer contents;
+   - |contents| : list of lines representing the buffer contents;
-   - |filetype|: buffer filetype. Empty string if no filetype is set;
+   - |filetype| : buffer filetype. Empty string if no filetype is set;
-   - |modified|: True if the buffer has unsaved changes, False otherwise;
+   - |modified| : True if the buffer has unsaved changes, False otherwise;
-   - |window|  : number of the buffer window. None if the buffer is hidden;
+   - |bufhidden|: value of the 'bufhidden' option (see :h bufhidden);
-   - |omnifunc|: omni completion function used by the buffer."""
+   - |window|   : number of the buffer window. None if the buffer is hidden;
+   - |omnifunc| : omni completion function used by the buffer."""
 
   def __init__( self, name,
                       number = 1,
                       contents = [],
                       filetype = '',
-                      modified = True,
+                      modified = False,
+                      bufhidden = '',
                       window = None,
                       omnifunc = '' ):
     self.name = os.path.realpath( name ) if name else ''
@@ -229,6 +236,7 @@ class VimBuffer( object ):
     self.contents = contents
     self.filetype = filetype
     self.modified = modified
+    self.bufhidden = bufhidden
     self.window = window
     self.omnifunc = omnifunc
     self.changedtick = 1
@@ -287,7 +295,7 @@ def MockVimBuffers( buffers, current_buffer, cursor_position = ( 1, 1 ) ):
   with patch( 'vim.buffers', buffers ):
     with patch( 'vim.current.buffer', current_buffer ):
       with patch( 'vim.current.window.cursor', cursor_position ):
-        yield
+        yield VIM_MOCK
 
 
 def MockVimModule():
@@ -319,6 +327,16 @@ def MockVimModule():
   return VIM_MOCK
 
 
+class VimError( Exception ):
+
+  def __init__( self, code ):
+      self.code = code
+
+
+  def __str__( self ):
+      return repr( self.code )
+
+
 class ExtendedMock( MagicMock ):
   """An extension to the MagicMock class which adds the ability to check that a
   callable is called with a precise set of calls in a precise order.

python/ycm/tests/vimsupport_test.py

@@ -27,7 +27,7 @@ from builtins import *  # noqa
 from ycm.tests import PathToTestFile
 from ycm.tests.test_utils import ( CurrentWorkingDirectory, ExtendedMock,
                                    MockVimBuffers, MockVimCommand,
-                                   MockVimModule, VimBuffer )
+                                   MockVimModule, VimBuffer, VimError )
 MockVimModule()
 
 from ycm import vimsupport
@@ -1602,3 +1602,178 @@ def EscapedFilepath_test():
        '/path/\ with\ /sp\ ac\ es' )
   eq_( vimsupport.EscapedFilepath( ' relative path/ with / spaces ' ),
        '\ relative\ path/\ with\ /\ spaces\ ' )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.command', new_callable = ExtendedMock )
+def JumpToLocation_SameFile_SameBuffer_NoSwapFile_test( vim_command ):
+  # No 'u' prefix for the current buffer name string to simulate Vim returning
+  # bytes on Python 2 but unicode on Python 3.
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  with MockVimBuffers( [ current_buffer ], current_buffer ) as vim:
+    vimsupport.JumpToLocation( os.path.realpath( u'uni¢𐍈d€' ), 2, 5 )
+
+    assert_that( vim.current.window.cursor, equal_to( ( 2, 4 ) ) )
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( 'normal! zz' )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.command', new_callable = ExtendedMock )
+def JumpToLocation_DifferentFile_SameBuffer_Unmodified_test( vim_command ):
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  with MockVimBuffers( [ current_buffer ], current_buffer ) as vim:
+    target_name = os.path.realpath( u'different_uni¢𐍈d€' )
+
+    vimsupport.JumpToLocation( target_name, 2, 5 )
+
+    assert_that( vim.current.window.cursor, equal_to( ( 2, 4 ) ) )
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( u'keepjumps edit {0}'.format( target_name ) ),
+      call( 'normal! zz' )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.command', new_callable = ExtendedMock )
+def JumpToLocation_DifferentFile_SameBuffer_Modified_CannotHide_test(
+    vim_command ):
+
+  current_buffer = VimBuffer( 'uni¢𐍈d€', modified = True )
+  with MockVimBuffers( [ current_buffer ], current_buffer ) as vim:
+    target_name = os.path.realpath( u'different_uni¢𐍈d€' )
+
+    vimsupport.JumpToLocation( target_name, 2, 5 )
+
+    assert_that( vim.current.window.cursor, equal_to( ( 2, 4 ) ) )
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( u'keepjumps split {0}'.format( target_name ) ),
+      call( 'normal! zz' )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.command', new_callable = ExtendedMock )
+def JumpToLocation_DifferentFile_SameBuffer_Modified_CanHide_test(
+    vim_command ):
+
+  current_buffer = VimBuffer( 'uni¢𐍈d€', modified = True, bufhidden = "hide" )
+  with MockVimBuffers( [ current_buffer ], current_buffer ) as vim:
+    target_name = os.path.realpath( u'different_uni¢𐍈d€' )
+
+    vimsupport.JumpToLocation( target_name, 2, 5 )
+
+    assert_that( vim.current.window.cursor, equal_to( ( 2, 4 ) ) )
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( u'keepjumps edit {0}'.format( target_name ) ),
+      call( 'normal! zz' )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.error', VimError )
+@patch( 'vim.command',
+        side_effect = [ None, VimError( 'Unknown code' ), None ] )
+def JumpToLocation_DifferentFile_SameBuffer_SwapFile_Unexpected_test(
+    vim_command ):
+
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  with MockVimBuffers( [ current_buffer ], current_buffer ):
+    assert_that(
+      calling( vimsupport.JumpToLocation ).with_args(
+          os.path.realpath( u'different_uni¢𐍈d€' ), 2, 5 ),
+      raises( VimError, 'Unknown code' )
+    )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.error', VimError )
+@patch( 'vim.command',
+        new_callable = ExtendedMock,
+        side_effect = [ None, VimError( 'E325' ), None ] )
+def JumpToLocation_DifferentFile_SameBuffer_SwapFile_Quit_test( vim_command ):
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  with MockVimBuffers( [ current_buffer ], current_buffer ):
+    target_name = os.path.realpath( u'different_uni¢𐍈d€' )
+
+    vimsupport.JumpToLocation( target_name, 2, 5 )
+
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( u'keepjumps edit {0}'.format( target_name ) )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'same-buffer' } )
+@patch( 'vim.error', VimError )
+@patch( 'vim.command',
+        new_callable = ExtendedMock,
+        side_effect = [ None, KeyboardInterrupt, None ] )
+def JumpToLocation_DifferentFile_SameBuffer_SwapFile_Abort_test( vim_command ):
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  with MockVimBuffers( [ current_buffer ], current_buffer ):
+    target_name = os.path.realpath( u'different_uni¢𐍈d€' )
+
+    vimsupport.JumpToLocation( target_name, 2, 5 )
+
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( u'keepjumps edit {0}'.format( target_name ) )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'new-or-existing-tab' } )
+@patch( 'vim.command', new_callable = ExtendedMock )
+def JumpToLocation_DifferentFile_NewOrExistingTab_NotAlreadyOpened_test(
+    vim_command ):
+
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  with MockVimBuffers( [ current_buffer ], current_buffer ):
+    target_name = os.path.realpath( u'different_uni¢𐍈d€' )
+
+    vimsupport.JumpToLocation( target_name, 2, 5 )
+
+    vim_command.assert_has_exact_calls( [
+      call( 'normal! m\'' ),
+      call( u'keepjumps tabedit {0}'.format( target_name ) ),
+      call( 'normal! zz' )
+    ] )
+
+
+@patch( 'ycmd.user_options_store._USER_OPTIONS',
+        { 'goto_buffer_command': 'new-or-existing-tab' } )
+@patch( 'vim.command', new_callable = ExtendedMock )
+def JumpToLocation_DifferentFile_NewOrExistingTab_AlreadyOpened_test(
+    vim_command ):
+
+  current_buffer = VimBuffer( 'uni¢𐍈d€' )
+  different_buffer = VimBuffer( 'different_uni¢𐍈d€' )
+  current_window = MagicMock( buffer = current_buffer )
+  different_window = MagicMock( buffer = different_buffer )
+  current_tab = MagicMock( windows = [ current_window, different_window ] )
+  with patch( 'vim.tabpages', [ current_tab ] ):
+    with MockVimBuffers( [ current_buffer, different_buffer ],
+                         current_buffer ) as vim:
+      vimsupport.JumpToLocation( os.path.realpath( u'different_uni¢𐍈d€' ),
+                                 2, 5 )
+
+      assert_that( vim.current.tabpage, equal_to( current_tab ) )
+      assert_that( vim.current.window, equal_to( different_window ) )
+      assert_that( vim.current.window.cursor, equal_to( ( 2, 4 ) ) )
+      vim_command.assert_has_exact_calls( [
+        call( 'normal! m\'' ),
+        call( 'normal! zz' )
+      ] )

python/ycm/vimsupport.py

@@ -152,7 +152,7 @@ def BufferIsVisible( buffer_number ):
 
 def GetBufferFilepath( buffer_object ):
   if buffer_object.name:
-    return buffer_object.name
+    return ToUnicode( buffer_object.name )
   # Buffers that have just been created by a command like :enew don't have any
   # buffer name so we use the buffer number for that.
   return os.path.join( GetCurrentDirectory(), str( buffer_object.number ) )
@@ -355,7 +355,9 @@ def VimExpressionToPythonType( vim_expression ):
 
 
 def HiddenEnabled( buffer_object ):
-  return bool( int( GetBufferOption( buffer_object, 'hid' ) ) )
+  if GetBufferOption( buffer_object, 'bh' ) == "hide":
+    return True
+  return GetBoolValue( '&hidden' )
 
 
 def BufferIsUsable( buffer_object ):
@@ -372,7 +374,7 @@ def TryJumpLocationInOpenedTab( filename, line, column ):
 
   for tab in vim.tabpages:
     for win in tab.windows:
-      if win.buffer.name == filepath:
+      if GetBufferFilepath( win.buffer ) == filepath:
         vim.current.tabpage = tab
         vim.current.window = win
         vim.current.window.cursor = ( line, column - 1 )