Pass the expected python version to the tests.

This ensure that we are actually testing against the python version that
we want to test again. A few weeks ago we still had the problem that all
our python3 tests ran against system python3 which was always 3.2. This
has been fixed a while ago, but this change makes sure we do not
regress.

Also fixes a couple of NOCOM comments that I left over in one of the
last commits.

test/vim_interface.py

@@ -107,6 +107,7 @@ class VimInterface(TempFileManager):
         raise NotImplementedError()
 
     def launch(self, config=[]):
+        """Returns the python version in Vim as a string, e.g. '2.7'"""
         pid_file = self.name_temp('vim.pid')
         done_file = self.name_temp('loading_done')
         if os.path.exists(done_file):
@@ -114,25 +115,25 @@ class VimInterface(TempFileManager):
 
         post_config = []
         post_config.append('%s << EOF' % ('py3' if PYTHON3 else 'py'))
-        post_config.append('import vim')
+        post_config.append('import vim, sys')
         post_config.append(
             "with open('%s', 'w') as pid_file: pid_file.write(vim.eval('getpid()'))" %
             pid_file)
-        post_config.append(
-            "with open('%s', 'w') as done_file: pass" %
-            done_file)
+        post_config.append("with open('%s', 'w') as done_file:" % done_file)
+        post_config.append("    done_file.write('%i.%i.%i' % sys.version_info[:3])")
         post_config.append('EOF')
 
         config_path = self.write_temp('vim_config.vim',
                                       textwrap.dedent(os.linesep.join(config + post_config) + '\n'))
 
         # Note the space to exclude it from shell history. Also we always set
-        # NVIM_LISTEN_ADDRESS, even when running vanilla vim, because it will
+        # NVIM_LISTEN_ADDRESS, even when running vanilla Vim, because it will
         # just not care.
         self.send_to_terminal(""" NVIM_LISTEN_ADDRESS=/tmp/nvim %s -u %s\r\n""" % (
             self._vim_executable, config_path))
         wait_until_file_exists(done_file)
-        self._vim_pid = int(open(pid_file, 'r').read())
+        self._vim_pid = int(read_text_file(pid_file))
+        return read_text_file(done_file).strip()
 
     def leave_with_wait(self):
         self.send_to_vim(3 * ESC + ':qa!\n')
@@ -198,8 +199,9 @@ class VimInterfaceTmuxNeovim(VimInterfaceTmux):
 
     def launch(self, config=[]):
         import neovim
-        VimInterfaceTmux.launch(self, config)
+        rv = VimInterfaceTmux.launch(self, config)
         self._nvim = neovim.attach('socket', path='/tmp/nvim')
+        return rv
 
 class VimInterfaceWindows(VimInterface):
     BRACES = re.compile('([}{])')

test/vim_test_case.py

@@ -34,12 +34,16 @@ class VimTestCase(unittest.TestCase, TempFileManager):
     version = None  # Will be set to vim --version output
     maxDiff = None  # Show all diff output, always.
     vim_flavor = None # will be 'vim' or 'neovim'.
+    expected_python_version = None # If set, we need to check that our Vim is running this python version.
 
     def __init__(self, *args, **kwargs):
         unittest.TestCase.__init__(self, *args, **kwargs)
         TempFileManager.__init__(self, 'Case')
 
     def runTest(self):
+        if self.expected_python_version:
+            self.assertEqual(self.in_vim_python_version, self.expected_python_version)
+
         # Only checks the output. All work is done in setUp().
         wanted = self.text_before + self.wanted + self.text_after
         if self.expected_error:
@@ -173,7 +177,7 @@ class VimTestCase(unittest.TestCase, TempFileManager):
         for name, content in self.files.items():
             self._create_file(name, content)
 
-        self.vim.launch(vim_config)
+        self.in_vim_python_version = self.vim.launch(vim_config)
 
         self._before_test()
 

test_all.py

@@ -107,11 +107,14 @@ if __name__ == '__main__':
                      help="Interface to use. Use 'tmux' with vanilla Vim and 'tmux_nvim' "
                      'with Neovim.')
         p.add_option('--python-host-prog', dest='python_host_prog', type=str, default='',
-                # NOCOM(#sirver): what
-                     help="")
+                     help='Neovim needs a variable to tell it which python interpretor to use for '
+                     'py blocks. This needs to be set to point to the correct python interpretor. '
+                     'It is ignored for vanilla Vim.')
         p.add_option('--python3-host-prog', dest='python3_host_prog', type=str, default='',
-                # NOCOM(#sirver): what
-                     help="")
+                     help='See --python-host-prog.')
+        p.add_option('--expected-python-version', dest='expected_python_version', type=str, default='',
+                     help='If set, each test will check sys.version inside of vim to '
+                     'verify we are testing against the expected Python version.')
 
         o, args = p.parse_args()
         return o, args
@@ -156,6 +159,7 @@ if __name__ == '__main__':
             test.test_plugins = options.plugins
             test.python_host_prog = options.python_host_prog
             test.python3_host_prog = options.python3_host_prog
+            test.expected_python_version = options.expected_python_version
             test.vim = vim
             test.vim_flavor = vim_flavor
             all_other_plugins.update(test.plugins)

travis_test.sh

@@ -32,9 +32,16 @@ else
    PY_IN_VIM="py3"
 fi
 
-echo "Using python from: $PYTHON_CMD Version: $($PYTHON_CMD --version 2>&1)"
+PYTHON_VERSION=$($PYTHON_CMD -c 'import sys;print(sys.version.split()[0])')
+echo "Using python from: $PYTHON_CMD Version: $PYTHON_VERSION"
 echo "Using vim from: $VIM. Version: $($VIMn)"
 
 tmux new -d -s vim
 
-$PYTHON_CMD ./test_all.py -v --plugins $INTERFACE --session vim --vim $VIM
+$PYTHON_CMD ./test_all.py \
+   -v \
+   --plugins \
+   --session vim \
+   --vim $VIM \
+   $INTERFACE \
+   --expected-python-version $PYTHON_VERSION