From 539c5576805f804b28c3504e0e39fc8d1e06a816 Mon Sep 17 00:00:00 2001 From: Holger Rapp Date: Sun, 1 Apr 2018 00:56:34 +0200 Subject: [PATCH] Move travis testing into docker for better consistency. (#950) This now only tests python2 on Vim 7.4, 8.0 and git. No Neovim (that has been broken for a long time anyways) and no longer any python3 testing. But hey, we have green tests again! --- .travis.yml | 29 ++++---- Dockerfile | 27 ++++++++ .../UltiSnips/snippet/definition/__init__.py | 2 +- .../{ultisnips.py => ulti_snips.py} | 2 +- .../parsing/{ultisnips.py => ulti_snips.py} | 0 pythonx/UltiSnips/snippet/source/__init__.py | 2 +- .../file/{ultisnips.py => ulti_snips.py} | 0 scripts/build_vim.sh | 27 ++++++++ scripts/download_vim.sh | 14 ++++ scripts/install_packages.sh | 11 +++ scripts/load_docker_cache.sh | 11 +++ scripts/run_tests.sh | 23 +++++++ scripts/save_docker_cache.sh | 14 ++++ test/vim_interface.py | 4 ++ test/vim_test_case.py | 3 +- travis_install.sh | 68 ------------------- travis_test.sh | 39 ----------- 17 files changed, 149 insertions(+), 127 deletions(-) create mode 100644 Dockerfile rename pythonx/UltiSnips/snippet/definition/{ultisnips.py => ulti_snips.py} (84%) rename pythonx/UltiSnips/snippet/parsing/{ultisnips.py => ulti_snips.py} (100%) rename pythonx/UltiSnips/snippet/source/file/{ultisnips.py => ulti_snips.py} (100%) create mode 100755 scripts/build_vim.sh create mode 100755 scripts/download_vim.sh create mode 100755 scripts/install_packages.sh create mode 100755 scripts/load_docker_cache.sh create mode 100755 scripts/run_tests.sh create mode 100755 scripts/save_docker_cache.sh delete mode 100755 travis_install.sh delete mode 100755 travis_test.sh diff --git a/.travis.yml b/.travis.yml index 082fa02..bdcff30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,24 +1,23 @@ -language: python +sudo: required +services: docker + +# Cache intermediate Docker layers. For a description of how this works, see: +# https://giorgos.sealabs.net/docker-cache-on-travis-and-docker-112.html +cache: + directories: + - /home/travis/docker/ -python: - - 2.7 - - 3.3 - - 3.4 env: - - VIM_VERSION="74" - - VIM_VERSION="git" - # - VIM_VERSION="NEOVIM" + - VIM_VERSION="7.4" PYTHON_IMAGE=2.7-stretch TAG=vim_74_py2 DOCKER_CACHE_FILE=/home/travis/docker/vim_74_py2.tar.gz + - VIM_VERSION="8.0" PYTHON_IMAGE=2.7-stretch TAG=vim_80_py2 DOCKER_CACHE_FILE=/home/travis/docker/vim_80_py2.tar.gz + - VIM_VERSION="git" PYTHON_IMAGE=2.7-stretch TAG=vim_git_py2 DOCKER_CACHE_FILE=/home/travis/docker/vim_git_py2.tar.gz install: - # Some of these commands fail transiently. We keep retrying them until they succeed. - - until sudo add-apt-repository ppa:kalakris/tmux -y; do sleep 10; done - - until sudo add-apt-repository ppa:neovim-ppa/unstable -y; do sleep 10; done - - until sudo apt-get update -qq; do sleep 10; done - - until sudo apt-get install -qq -y --force-yes tmux xclip gdb neovim git; do sleep 10; done - - ./travis_install.sh + - docker build -t ultisnips:${TAG} --build-arg PYTHON_IMAGE=${PYTHON_IMAGE} --build-arg VIM_VERSION=${VIM_VERSION} . script: - - ./travis_test.sh + - docker run -it ultisnips:${TAG} scripts/run_tests.sh + - scripts/save_docker_cache.sh notifications: webhooks: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c7b697b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +ARG PYTHON_IMAGE + +FROM python:${PYTHON_IMAGE} + +ARG VIM_VERSION + +COPY scripts/install_packages.sh src/scripts/ +RUN src/scripts/install_packages.sh +COPY scripts/download_vim.sh src/scripts/ +RUN src/scripts/download_vim.sh +COPY scripts/build_vim.sh src/scripts/ +RUN src/scripts/build_vim.sh + +# We clone the plugins we currently depend on manually here. Initially we check if their master +# has changed since last time we build, this will invalidate the cache. +RUN mkdir -p /tmp/UltiSnips_test_vim_plugins + +ADD https://api.github.com/repos/tpope/vim-pathogen/git/refs/heads/master \ + /src/scripts/vim-pathogen_version.json +RUN git clone --recursive --depth 1 https://github.com/tpope/vim-pathogen /tmp/UltiSnips_test_vim_plugins/vim-pathogen + +ADD https://api.github.com/repos/ervandew/supertab/git/refs/heads/master \ + /src/scripts/supertab_version.json +RUN git clone --recursive --depth 1 https://github.com/ervandew/supertab /tmp/UltiSnips_test_vim_plugins/supertab + +COPY . /src/UltiSnips +WORKDIR /src/UltiSnips diff --git a/pythonx/UltiSnips/snippet/definition/__init__.py b/pythonx/UltiSnips/snippet/definition/__init__.py index fb08801..ee1989a 100644 --- a/pythonx/UltiSnips/snippet/definition/__init__.py +++ b/pythonx/UltiSnips/snippet/definition/__init__.py @@ -1,4 +1,4 @@ """In memory representation of snippet definitions.""" -from UltiSnips.snippet.definition.ultisnips import UltiSnipsSnippetDefinition +from UltiSnips.snippet.definition.ulti_snips import UltiSnipsSnippetDefinition from UltiSnips.snippet.definition.snipmate import SnipMateSnippetDefinition diff --git a/pythonx/UltiSnips/snippet/definition/ultisnips.py b/pythonx/UltiSnips/snippet/definition/ulti_snips.py similarity index 84% rename from pythonx/UltiSnips/snippet/definition/ultisnips.py rename to pythonx/UltiSnips/snippet/definition/ulti_snips.py index 5338a6a..75871b6 100644 --- a/pythonx/UltiSnips/snippet/definition/ultisnips.py +++ b/pythonx/UltiSnips/snippet/definition/ulti_snips.py @@ -4,7 +4,7 @@ """A UltiSnips snippet after parsing.""" from UltiSnips.snippet.definition._base import SnippetDefinition -from UltiSnips.snippet.parsing.ultisnips import parse_and_instantiate +from UltiSnips.snippet.parsing.ulti_snips import parse_and_instantiate class UltiSnipsSnippetDefinition(SnippetDefinition): diff --git a/pythonx/UltiSnips/snippet/parsing/ultisnips.py b/pythonx/UltiSnips/snippet/parsing/ulti_snips.py similarity index 100% rename from pythonx/UltiSnips/snippet/parsing/ultisnips.py rename to pythonx/UltiSnips/snippet/parsing/ulti_snips.py diff --git a/pythonx/UltiSnips/snippet/source/__init__.py b/pythonx/UltiSnips/snippet/source/__init__.py index 08c20ac..65698c4 100644 --- a/pythonx/UltiSnips/snippet/source/__init__.py +++ b/pythonx/UltiSnips/snippet/source/__init__.py @@ -6,5 +6,5 @@ from UltiSnips.snippet.source._base import SnippetSource from UltiSnips.snippet.source.added import AddedSnippetsSource from UltiSnips.snippet.source.file.snipmate import SnipMateFileSource -from UltiSnips.snippet.source.file.ultisnips import UltiSnipsFileSource, \ +from UltiSnips.snippet.source.file.ulti_snips import UltiSnipsFileSource, \ find_all_snippet_files, find_snippet_files diff --git a/pythonx/UltiSnips/snippet/source/file/ultisnips.py b/pythonx/UltiSnips/snippet/source/file/ulti_snips.py similarity index 100% rename from pythonx/UltiSnips/snippet/source/file/ultisnips.py rename to pythonx/UltiSnips/snippet/source/file/ulti_snips.py diff --git a/scripts/build_vim.sh b/scripts/build_vim.sh new file mode 100755 index 0000000..2087da1 --- /dev/null +++ b/scripts/build_vim.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o errexit +set -o verbose + +cd /src/vim + +if [[ $PYTHON_VERSION =~ ^2\. ]]; then + PYTHON_BUILD_CONFIG="--enable-pythoninterp" +else + PYTHON_BUILD_CONFIG="--enable-python3interp" +fi + +export CFLAGS="$(python-config --cflags)" +echo $CFLAGS +./configure \ + --prefix=${HOME} \ + --disable-nls \ + --disable-sysmouse \ + --disable-gpm \ + --enable-gui=no \ + --enable-multibyte \ + --with-features=huge \ + --with-tlib=ncurses \ + --without-x \ + $PYTHON_BUILD_CONFIG || cat $(find . -name 'config.log') +make install diff --git a/scripts/download_vim.sh b/scripts/download_vim.sh new file mode 100755 index 0000000..0ad1767 --- /dev/null +++ b/scripts/download_vim.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -o errexit +set -o verbose + +mkdir -p /src && cd /src + +if [[ $VIM_VERSION == "git" ]]; then + git clone https://github.com/vim/vim +else + curl http://ftp.vim.org/pub/vim/unix/vim-${VIM_VERSION}.tar.bz2 -o vim.tar.bz2 + tar xjf vim.tar.bz2 + mv -v vim?? vim +fi diff --git a/scripts/install_packages.sh b/scripts/install_packages.sh new file mode 100755 index 0000000..8727139 --- /dev/null +++ b/scripts/install_packages.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -o errexit +set -o verbose + +apt-get update +apt-get install -y \ + g++ \ + tmux \ + git +apt-get clean diff --git a/scripts/load_docker_cache.sh b/scripts/load_docker_cache.sh new file mode 100755 index 0000000..5e243ce --- /dev/null +++ b/scripts/load_docker_cache.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o errexit +set -o verbose +set -o pipefail + +# See +# https://giorgos.sealabs.net/docker-cache-on-travis-and-docker-112.html +if [ -f ${DOCKER_CACHE_FILE} ]; then + gunzip -c ${DOCKER_CACHE_FILE} | docker load; +fi diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100755 index 0000000..371befd --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +PYTHON_CMD="$(which python)" +INTERFACE="--interface tmux" +VIM="$HOME/bin/vim" +PYTHON_VERSION=$($PYTHON_CMD -c 'import sys;print(sys.version.split()[0])') +echo "Using python from: $PYTHON_CMD Version: $PYTHON_VERSION" +echo "Using vim from:" +$VIM --version | head -n 3 + +set -x + +tmux new -d -s vim + +$PYTHON_CMD ./test_all.py \ + -v \ + --plugins \ + --session vim \ + --vim $VIM \ + $INTERFACE \ + --expected-python-version $PYTHON_VERSION diff --git a/scripts/save_docker_cache.sh b/scripts/save_docker_cache.sh new file mode 100755 index 0000000..38c13a6 --- /dev/null +++ b/scripts/save_docker_cache.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -o errexit +set -o verbose +set -o pipefail + +# Cache intermediate Docker layers. For a description of how this works, see: +# https://giorgos.sealabs.net/docker-cache-on-travis-and-docker-112.html +if [[ ${TRAVIS_BRANCH} == "master" ]] && [[ ${TRAVIS_PULL_REQUEST} == "false" ]]; then + mkdir -p $(dirname ${DOCKER_CACHE_FILE}) + IMAGE_NAMES=$(docker history -q ultisnips:${TAG} | grep -v '') + docker save ${IMAGE_NAMES} | gzip > ${DOCKER_CACHE_FILE}.new + mv ${DOCKER_CACHE_FILE}.new ${DOCKER_CACHE_FILE} +fi diff --git a/test/vim_interface.py b/test/vim_interface.py index c885589..e763c6c 100644 --- a/test/vim_interface.py +++ b/test/vim_interface.py @@ -93,6 +93,10 @@ class VimInterface(TempFileManager): self._vim_executable = vim_executable self._version = None + @property + def vim_executable(self): + return self._vim_executable + def has_version(self, major, minor, patchlevel): if self._version is None: output = subprocess.check_output([ diff --git a/test/vim_test_case.py b/test/vim_test_case.py index b689bbb..4d91c37 100644 --- a/test/vim_test_case.py +++ b/test/vim_test_case.py @@ -83,9 +83,8 @@ class VimTestCase(unittest.TestCase, TempFileManager): os.symlink(source, os.path.join(absdir, os.path.basename(source))) def setUp(self): - # TODO(sirver): this uses 'vim', but must use --vim from the commandline. if not VimTestCase.version: - VimTestCase.version, _ = subprocess.Popen(['vim', '--version'], + VimTestCase.version, _ = subprocess.Popen([self.vim.vim_executable, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() if PYTHON3: VimTestCase.version = VimTestCase.version.decode('utf-8') diff --git a/travis_install.sh b/travis_install.sh deleted file mode 100755 index da42e38..0000000 --- a/travis_install.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# Installs a known version of vim in the travis test runner. - -set -ex - -PYTHON="python${TRAVIS_PYTHON_VERSION}" - -build_vanilla_vim () { - mkdir ~/vim_build - pushd ~/vim_build - - if [[ $VIM_VERSION == "74" ]]; then - until curl ftp://ftp.vim.org/pub/vim/unix/vim-7.4.tar.bz2 -o vim.tar.bz2; do sleep 10; done - tar xjf vim.tar.bz2 - cd vim${VIM_VERSION} - elif [[ $VIM_VERSION == "git" ]]; then - git clone https://github.com/vim/vim - cd vim - fi - - local PYTHON_CONFIG_DIR=$(dirname $(find $($PYTHON-config --prefix)/lib -iname 'config.c') | grep $TRAVIS_PYTHON_VERSION) - local PYTHON_BUILD_CONFIG="" - if [[ $TRAVIS_PYTHON_VERSION =~ ^2\. ]]; then - PYTHON_BUILD_CONFIG="--enable-pythoninterp --with-python-config-dir=${PYTHON_CONFIG_DIR}" - else - PYTHON_BUILD_CONFIG="--enable-python3interp --with-python3-config-dir=${PYTHON_CONFIG_DIR}" - fi - export LDFLAGS="$($PYTHON-config --ldflags) -L$($PYTHON-config --prefix)/lib" - export CFLAGS="$($PYTHON-config --cflags)" - - # This is needed so that vim finds the shared libraries it was build against - # - they are not on the regular path. - export LD_LIBRARY_PATH="$($PYTHON-config --prefix)/lib" - - echo $LDFLAGS - echo $CFLAGS - ./configure \ - --prefix=${HOME} \ - --disable-nls \ - --disable-sysmouse \ - --disable-gpm \ - --enable-gui=no \ - --enable-multibyte \ - --with-features=huge \ - --with-tlib=ncurses \ - --without-x \ - ${PYTHON_BUILD_CONFIG} || cat $(find . -name 'config.log') - - make install - popd - - rm -rf vim_build -} - -if [[ $VIM_VERSION = "74" || $VIM_VERSION = "git" ]]; then - build_vanilla_vim -elif [[ $VIM_VERSION == "NEOVIM" ]]; then - PIP=$(which pip) - $PIP install neovim -else - echo "Unknown VIM_VERSION: $VIM_VERSION" - exit 1 -fi - -# Clone the dependent plugins we want to use. -PYTHON_CMD="$(which $PYTHON)" -$PYTHON_CMD ./test_all.py --clone-plugins diff --git a/travis_test.sh b/travis_test.sh deleted file mode 100755 index d9dcd46..0000000 --- a/travis_test.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -PYTHON="python${TRAVIS_PYTHON_VERSION}" -PYTHON_CMD="$(which ${PYTHON})" - -if [[ $VIM_VERSION = "74" || $VIM_VERSION = "git" ]]; then - INTERFACE="--interface tmux" - VIM="${HOME}/bin/vim" - # This is needed so that vim finds the shared libraries it was build against - - # they are not on the regular path. - export LD_LIBRARY_PATH="$($PYTHON-config --prefix)/lib" - -elif [[ $VIM_VERSION == "NEOVIM" ]]; then - VIM="$(which nvim)" - if [[ $TRAVIS_PYTHON_VERSION =~ ^2\. ]]; then - INTERFACE="--interface tmux_nvim --python-host-prog=$PYTHON_CMD" - else - INTERFACE="--interface tmux_nvim --python3-host-prog=$PYTHON_CMD" - fi -else - echo "Unknown VIM_VERSION: $VIM_VERSION" - exit 1 -fi - -PYTHON_VERSION=$($PYTHON_CMD -c 'import sys;print(sys.version.split()[0])') -echo "Using python from: $PYTHON_CMD Version: $PYTHON_VERSION" -echo "Using vim from: $VIM. Version: $($VIMn)" - -tmux new -d -s vim - -$PYTHON_CMD ./test_all.py \ - -v \ - --plugins \ - --session vim \ - --vim $VIM \ - $INTERFACE \ - --expected-python-version $PYTHON_VERSION