diff --git a/Makefile b/Makefile index 0d01896..f454324 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ test: done; \ exit $$result +quiet-test: + $(MAKE) test QUIET=y + perf: @result=0; \ for test in highlighters/*; do \ diff --git a/tests/README.md b/tests/README.md index faaed1e..2674aa7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -38,6 +38,9 @@ All tests may be run with make test which will run all highlighting tests and report results in [TAP format][TAP]. +By default, the results of all tests will be printed; to show only "interesting" +results (tests that failed but were expected to succeed, or vice-versa), run +`make quiet-test` (or `make test QUIET=y`). [TAP]: http://testanything.org/ diff --git a/tests/tap-filter b/tests/tap-filter new file mode 100755 index 0000000..6587808 --- /dev/null +++ b/tests/tap-filter @@ -0,0 +1,45 @@ +#!/usr/bin/env perl +# ------------------------------------------------------------------------------------------------- +# Copyright (c) 2015 zsh-syntax-highlighting contributors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are permitted +# provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this list of conditions +# and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, this list of +# conditions and the following disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors +# may be used to endorse or promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ------------------------------------------------------------------------------------------------- +# vim: ft=perl sw=2 ts=2 et +# ------------------------------------------------------------------------------------------------- + +# This is a stdin-to-stdout filter that takes TAP output (such as 'make test') +# on stdin and deletes lines pertaining to expected results. +# +# More specifically, if any of the test points in a test file either failed but +# was expected to pass, or passed but was expected to fail, then emit that test +# file's output; else, elide that test file's output. + +use v5.10.0; +use warnings; +use strict; + +undef $/; # slurp mode +print for + grep { /^ok.*# TODO/m or /^not ok(?!.*# TODO)/m } + split /^(?=#)/m, + ; diff --git a/tests/test-highlighting.zsh b/tests/test-highlighting.zsh index 5d35452..dc47989 100755 --- a/tests/test-highlighting.zsh +++ b/tests/test-highlighting.zsh @@ -132,10 +132,23 @@ run_test() { } } +# Set up results_filter +local results_filter +if [[ $QUIET == y ]]; then + if type -w perl >/dev/null; then + results_filter=${0:A:h}/tap-filter + else + echo >&2 "Bail out! quiet mode not supported: perl not found"; exit 2 + fi +else + results_filter=cat +fi +[[ -n $results_filter ]] || { echo >&2 "Bail out! BUG setting \$results_filter"; exit 2 } + # Process each test data file in test data directory. integer something_failed=0 for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do - run_test "$data_file" | tee >(${0:A:h}/tap-colorizer.zsh) | grep -v '^not ok.*# TODO' | grep -q '^not ok\|^ok.*# TODO' && (( something_failed=1 )) + run_test "$data_file" | tee >($results_filter | ${0:A:h}/tap-colorizer.zsh) | grep -v '^not ok.*# TODO' | grep -q '^not ok\|^ok.*# TODO' && (( something_failed=1 )) (( $pipestatus[1] )) && exit 2 done