ci: Refactor CI scripts

1. CI_TARGET now determines which run_${CI_TARGET}.sh script to use. Defaults to
   `tests`.
2. Build no longer halts on the first failing suit: e.g. if functional tests
   failed it will continue with unit tests, etc.
3. All ${MAKE_CMD} occurrences moved to `top_make` function, added `build_make`
   as an alias to `make -C build` (`"${MAKE_CMD}" -C "${BUILD_DIR}"`) which is
   too verbose.

`suite.sh` was copied from powerline (tests/common.sh file), assumes running
with POSIX shells (and actually uses dash in powerline). Then some convenience
functions were added (run_test and below).

.travis.yml

@@ -50,6 +50,7 @@ env:
     - SUCCESS_MARKER="$BUILD_DIR/.tests_successful"
     # default target name for functional tests
     - FUNCTIONALTEST=functionaltest
+    - CI_TARGET=tests
 
 matrix:
   include:

ci/common/build.sh

@@ -1,3 +1,11 @@
+top_make() {
+  "${MAKE_CMD}" "$@"
+}
+
+build_make() {
+  top_make -C "${BUILD_DIR}" "$@"
+}
+
 build_deps() {
   if [[ "${BUILD_32BIT}" == ON ]]; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
@@ -30,7 +38,7 @@ build_deps() {
   echo "Configuring with '${DEPS_CMAKE_FLAGS}'."
   CC= cmake ${DEPS_CMAKE_FLAGS} "${TRAVIS_BUILD_DIR}/third-party/"
 
-  if ! ${MAKE_CMD}; then
+  if ! top_make; then
     exit 1
   fi
 
@@ -53,18 +61,18 @@ prepare_build() {
 
 build_nvim() {
   echo "Building nvim."
-  if ! ${MAKE_CMD} nvim; then
+  if ! top_make nvim; then
     exit 1
   fi
 
   if [ "$CLANG_SANITIZER" != "TSAN" ]; then
     echo "Building libnvim."
-    if ! ${MAKE_CMD} libnvim; then
+    if ! top_make libnvim; then
       exit 1
     fi
 
     echo "Building nvim-test."
-    if ! ${MAKE_CMD} nvim-test; then
+    if ! top_make nvim-test; then
       exit 1
     fi
   fi

ci/common/suite.sh

@@ -0,0 +1,55 @@
+FAILED=0
+
+FAIL_SUMMARY=""
+
+enter_suite() {
+  local suite_name="$1"
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
+}
+
+exit_suite() {
+  if test $FAILED -ne 0 ; then
+    echo "Suite ${NVIM_TEST_CURRENT_SUITE} failed, summary:"
+    echo "${FAIL_SUMMARY}"
+  fi
+  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
+  if test "x$1" != "x--continue" ; then
+    exit $FAILED
+  fi
+}
+
+fail() {
+  local allow_failure=
+  if test "x$1" = "x--allow-failure" ; then
+    shift
+    allow_failure=A
+  fi
+  local test_name="$1"
+  local fail_char="$allow_failure$2"
+  local message="$3"
+
+  : ${fail_char:=F}
+  : ${message:=Test $test_name failed}
+
+  local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
+  FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  echo "Failed: $full_msg"
+  if test "x$allow_failure" = "x" ; then
+    FAILED=1
+  fi
+}
+
+run_test() {
+  local cmd="$1"
+  shift
+  local test_name="$1"
+  : ${test_name:=$cmd}
+  shift
+  if ! eval "$cmd" ; then
+    fail "${test_name}" "$@"
+  fi
+}
+
+succeeded() {
+  return $FAILED
+}

ci/common/test.sh

@@ -1,3 +1,5 @@
+source "${CI_DIR}/common/build.sh"
+
 print_core() {
   local app="$1"
   local core="$2"
@@ -75,7 +77,7 @@ asan_check() {
 
 run_unittests() {
   ulimit -c unlimited
-  if ! ${MAKE_CMD} -C "${BUILD_DIR}" unittest ; then
+  if ! build_make unittest ; then
     check_core_dumps "$(which luajit)"
     exit 1
   fi
@@ -84,7 +86,7 @@ run_unittests() {
 
 run_functionaltests() {
   ulimit -c unlimited
-  if ! ${MAKE_CMD} -C "${BUILD_DIR}" ${FUNCTIONALTEST}; then
+  if ! build_make ${FUNCTIONALTEST}; then
     asan_check "${LOG_DIR}"
     valgrind_check "${LOG_DIR}"
     check_core_dumps
@@ -110,7 +112,7 @@ run_oldtests() {
 }
 
 install_nvim() {
-  ${MAKE_CMD} -C "${BUILD_DIR}" install
+  build_make install
 
   "${INSTALL_PREFIX}/bin/nvim" --version
   "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {

ci/run_lint.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/suite.sh"
+
+enter_suite 'lint'
+
+run_test 'top_make clint-full' clint
+run_test 'top_make testlint' testlint
+run_test 'top_make check-single-includes' single-includes
+
+exit_suite

ci/run_tests.sh

@@ -6,6 +6,9 @@ set -o pipefail
 CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
+source "${CI_DIR}/common/suite.sh"
+
+enter_suite tests
 
 check_core_dumps --delete quiet
 
@@ -15,11 +18,15 @@ build_nvim
 if [ "$CLANG_SANITIZER" != "TSAN" ]; then
   # Additional threads are only created when the builtin UI starts, which
   # doesn't happen in the unit/functional tests
-  run_unittests
+  run_test run_unittests
-  run_functionaltests
+  run_test run_functionaltests
 fi
-run_oldtests
+run_test run_oldtests
 
-install_nvim
+run_test install_nvim
 
-touch "${SUCCESS_MARKER}"
+if succeeded ; then
+  touch "${SUCCESS_MARKER}"
+fi
+
+exit_suite

ci/script.sh

@@ -3,16 +3,11 @@
 set -e
 set -o pipefail
 
-if [[ -n "${CI_TARGET}" ]]; then
-  make "${CI_TARGET}"
-  exit 0
-fi
-
 # This will pass the environment variables down to a bash process which runs
 # as $USER, while retaining the environment variables defined and belonging
 # to secondary groups given above in usermod.
 if [[ "${TRAVIS_OS_NAME}" == osx ]]; then
-  sudo -E su "${USER}" -c "ci/run_tests.sh"
+  sudo -E su "${USER}" -c "ci/run_${CI_TARGET}.sh"
 else
-  ci/run_tests.sh
+  ci/run_${CI_TARGET}.sh
 fi