Fixes for Unicode issues by jszakmeister.

pythonx/UltiSnips/compatibility.py

@@ -72,6 +72,8 @@ else:
         """Encode 'string' using &encoding."""
         try:
             return string.encode(vim.eval("&encoding"))
+        except UnicodeDecodeError:
+            return string
         except UnicodeEncodeError:
             return string
 

pythonx/UltiSnips/vim_state.py

@@ -31,27 +31,34 @@ class VimState(object):
         self._poss = deque(maxlen=5)
         self._lvb = None
 
-        self._text_to_expect = None
-        self._unnamed_reg_cache = None
+        self._text_to_expect = ""
         self._unnamed_reg_cached = False
 
+        # We store the cached value of the unnamed register in Vim directly to
+        # avoid any Unicode issues with saving and restoring the unnamed
+        # register across the Python bindings.  The unnamed register can contain
+        # data that cannot be coerced to Unicode, and so a simple vim.eval('@"')
+        # fails badly.  Keeping the cached value in Vim directly, sidesteps the
+        # problem.
+        _vim.command('let g:_ultisnips_unnamed_reg_cache = ""')
+
     def remember_unnamed_register(self, text_to_expect):
         """Save the unnamed register. 'text_to_expect' is text that we expect
         to be contained in the register the next time this method is called -
         this could be text from the tabstop that was selected and might have
         been overwritten. We will not cash that then."""
         self._unnamed_reg_cached = True
-        unnamed_reg = _vim.eval('@"')
-        if unnamed_reg != self._text_to_expect:
-            self._unnamed_reg_cache = unnamed_reg
+        escaped_text = self._text_to_expect.replace("'", "''")
+        res = int(_vim.eval('@" != ' + "'" + escaped_text + "'"))
+        if res:
+            _vim.command('let g:_ultisnips_unnamed_reg_cache = @"')
         self._text_to_expect = text_to_expect
 
     def restore_unnamed_register(self):
         """Restores the unnamed register and forgets what we cached."""
         if not self._unnamed_reg_cached:
             return
-        escaped_cache = self._unnamed_reg_cache.replace("'", "''")
-        _vim.command("let @\"='%s'" % escaped_cache)
+        _vim.command('let @" = g:_ultisnips_unnamed_reg_cache')
         self._unnamed_reg_cached = False
 
     def remember_position(self):

test/test_Fixes.py

@@ -44,3 +44,27 @@ class ShiftWidthZero(_VimTest):
   keys = "test" + EX + "foo"
   wanted = "\tfoo"
 
+# Test for https://github.com/SirVer/ultisnips/issues/171 {{{#
+# Make sure that we don't crash when trying to save and restore the clipboard
+# when it contains data that we can't coerce into Unicode.
+class NonUnicodeDataInUnnamedRegister(_VimTest):
+    snippets = ("test", "hello")
+    keys = "test" + EX + ESC + \
+            "\n".join([":redir @a",
+                       ":messages",
+                       ":redir END",
+                       (":if match(@a, 'Error') != -1 | " +
+                        "call setline('.', 'error detected') | " +
+                        "3put a | " +
+                        "endif"),
+                       ""])
+    wanted = "hello"
+
+    def _before_test(self):
+        # The string below was the one a user had on their clipboard when
+        # encountering the UnicodeDecodeError and could not be coerced into
+        # unicode.
+        self.vim.send(
+                ':let @" = "\\x80kdI{\\x80@7 1},' +
+                '\\x80kh\\x80kh\\x80kd\\x80kdq\\x80kb\\x1b"\n')
+# End: #171  #}}}

test/vim_test_case.py

@@ -357,6 +357,7 @@ class VimTestCase(unittest.TestCase, TempFileManager):
         vim_config.append('set fileencoding=utf-8')
         vim_config.append('set buftype=nofile')
         vim_config.append('set shortmess=at')
+        vim_config.append('let @" = ""')
         vim_config.append('let g:UltiSnipsExpandTrigger="<tab>"')
         vim_config.append('let g:UltiSnipsJumpForwardTrigger="?"')
         vim_config.append('let g:UltiSnipsJumpBackwardTrigger="+"')