oldtests: Run old tests in :terminal

Reasoning:

- No need to check for terminal size.
- No need to mess with terminal title.
- Allows old tests to timeout with the rest of the build proceeding.
- Less and less messy output to travis log.
- Opens a path allowing old tests run in parallel. Though last bit needs test
  refactoring.

src/nvim/testdir/Makefile

@@ -7,7 +7,7 @@ ifeq ($(OS),Windows_NT)
 else
   NVIM_PRG ?= ../../../build/bin/nvim
 endif
-SCRIPTSOURCE := ../../../runtime
+ROOT := ../../..
 
 export SHELL := sh
 export NVIM_PRG := $(NVIM_PRG)
@@ -134,7 +134,7 @@ ifdef USE_VALGRIND
 	                 $(VALGRIND_TOOL)                       \
 	                 --suppressions=../../.valgrind.supp \
 	                 --error-exitcode=123                   \
-	                 --log-file=valgrind.\%p.$*             \
+	                 --log-file=valgrind-\%p.$*             \
 	                 $(VGDB)                                \
 	                 --trace-children=yes
 else
@@ -171,7 +171,7 @@ $(SCRIPTS) $(SCRIPTS_GUI): $(NVIM_PRG) test1.out
 
 RM_ON_RUN   := test.out X* viminfo
 RM_ON_START := test.ok
-RUN_VIM     := VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE -i viminfo --headless --noplugin -s dotest.in
+RUN_VIM     := $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE -i viminfo --headless --noplugin -s dotest.in
 
 clean:
 	-rm -rf *.out          \
@@ -179,6 +179,7 @@ clean:
 	        *.res          \
 	        *.rej          \
 	        *.orig         \
+	        *.tlog         \
 	        test.log       \
 	        messages       \
 	        $(RM_ON_RUN)   \
@@ -193,45 +194,16 @@ clean:
 test1.out: .gdbinit test1.in
 	-rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
 	mkdir -p $(TMPDIR)
-	$(RUN_VIM) $*.in
+	/bin/sh runnvim.sh $(ROOT) $(NVIM_PRG) $* $(RUN_VIM) $*.in
-	@/bin/sh -c "if test -e wrongtermsize; then                                 \
+	-rm -f wrongtermsize
-	                 echo;                                                      \
-	                 echo test1 FAILED - terminal size must be 80x24 or larger; \
-	                 echo; exit 1;                                              \
-	             elif diff test.out $*.ok; then                                 \
-	                 mv -f test.out $*.out;                                     \
-	             else                                                           \
-	                 echo;                                                      \
-	                 echo test1 FAILED - Something basic is wrong;              \
-	                 echo;                                                      \
-	                 exit 1;                                                    \
-	             fi"
 	-rm -rf X* viminfo
 
 %.out: %.in .gdbinit
 	-rm -rf $*.failed test.ok $(RM_ON_RUN)
 	mkdir -p $(TMPDIR)
 	cp $*.ok test.ok
-	# Sleep a moment to avoid that the xterm title is messed up.
+	/bin/sh runnvim.sh --oldesttest $(ROOT) $(NVIM_PRG) $* $(RUN_VIM) $*.in
-	# 200 msec is sufficient, but only modern sleep supports a fraction of
-	# a second, fall back to a second if it fails.
-	@-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
-	$(RUN_VIM) $*.in
 
-	# Check if the test.out file matches test.ok.
-	@/bin/sh -c "if test -f test.out; then                \
-	                 if diff -u test.out $*.ok; then         \
-	                     mv -f test.out $*.out;           \
-	                 else                                 \
-	                     echo $* FAILED >> test.log;      \
-	                     mv -f test.out $*.failed;        \
-	                 fi;                                  \
-	             else                                     \
-	                 echo $* NO OUTPUT >>test.log;        \
-	             fi"
-	@/bin/sh -c "if test -f valgrind; then                \
-	                 mv -f valgrind valgrind.$*;          \
-	             fi"
 	-rm -rf X* test.ok viminfo
 
 test49.out: test49.vim
@@ -243,7 +215,7 @@ nolog:
 # New style of tests uses Vim script with assert calls.  These are easier
 # to write and a lot easier to read and debug.
 # Limitation: Only works with the +eval feature.
-RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE --headless --noplugin
+RUN_VIMTEST = $(TOOL) $(NVIM_PRG) -u unix.vim -U NONE --headless --noplugin
 
 newtests: newtestssilent
 	@/bin/sh -c "if test -f messages && grep -q 'FAILED' messages; then \
@@ -254,4 +226,4 @@ newtestssilent: $(NEW_TESTS)
 
 %.res: %.vim .gdbinit
 	mkdir -p $(TMPDIR)
-	$(RUN_VIMTEST) -u NONE -S runtest.vim $*.vim
+	/bin/sh runnvim.sh $(ROOT) $(NVIM_PRG) $* $(RUN_VIMTEST) -u NONE -S runtest.vim $*.vim

src/nvim/testdir/runnvim.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+
+main() {(
+  local oldesttest=
+  if test "$1" = "--oldesttest" ; then
+    shift
+    oldesttest=1
+  fi
+  local root="$1" ; shift
+  local nvim_prg="$1" ; shift
+  local test_name="$1" ; shift
+
+  local tlog="$test_name.tlog"
+
+  export NVIM_TEST_ARGC=$#
+  local arg
+  local i=0
+  for arg ; do
+    eval "export NVIM_TEST_ARG$i=\"\$arg\""
+    i=$(( i+1 ))
+  done
+
+  export CI_DIR="$root/ci"
+  export BUILD_DIR="$(dirname "$nvim_prg")/.."
+  export FAILED=0
+
+  . "$CI_DIR/common/suite.sh"
+  . "$CI_DIR/common/test.sh"
+
+  export VIMRUNTIME="$root/runtime"
+  if ! "$nvim_prg" \
+    -u NONE -i NONE \
+    --headless \
+    --cmd 'set shortmess+=I noswapfile noundofile nomore' \
+    -S runnvim.vim \
+    "$tlog"
+  then
+    fail "$test_name" F "Nvim exited with non-zero code"
+  fi
+  if test "$oldesttest" = 1 ; then
+    if ! diff -q test.out "$test_name.ok" > /dev/null 2>&1 ; then
+      if test -f test.out ; then
+        fail "$test_name" F "Oldest test .out file differs from .ok file"
+        echo "Diff between test.out and $test_name.ok" >> "$tlog"
+        echo "================================================================================" >> "$tlog"
+        diff -a test.out "$test_name.ok" >> "$tlog"
+        echo "================================================================================" >> "$tlog"
+      else
+        echo "No output in test.out" >> "$tlog"
+      fi
+    fi
+  fi
+  if test "$FAILED" = 1 ; then
+    travis_fold start "$NVIM_TEST_CURRENT_SUITE/$test_name"
+  fi
+  valgrind_check .
+  if test -n "$LOG_DIR" ; then
+    asan_check "$LOG_DIR"
+  fi
+  check_core_dumps
+  if test "$FAILED" = 1 ; then
+    cat "$tlog"
+  fi
+  rm -f "$tlog"
+  if test "$FAILED" = 1 ; then
+    travis_fold end "$NVIM_TEST_CURRENT_SUITE/$test_name"
+  fi
+  if test "$FAILED" = 1 ; then
+    echo "Test $test_name failed, see output above and summary for more details" >> test.log
+  fi
+)}
+
+main "$@"

src/nvim/testdir/runnvim.vim

@@ -0,0 +1,55 @@
+let s:logger = {'d_events': []}
+function s:logger.on_stdout(id, data, event)
+  call add(self.d_events, [a:event, a:data])
+endfunction
+let s:logger.on_stderr = s:logger.on_stdout
+function s:logger.on_exit(id, data, event)
+  call add(self.d_events, [a:event, ['']])
+endfunction
+
+function Main()
+  let argc = +$NVIM_TEST_ARGC
+  let args = []
+  for i in range(argc)
+    call add(args, eval("$NVIM_TEST_ARG" . i))
+  endfor
+  set lines=25
+  set columns=80
+  enew
+  let job = termopen(args, s:logger)
+  let results = jobwait([job], 5 * 60 * 1000)
+  " TODO(ZyX-I): Get colors
+  let screen = getline(1, '$')
+  bwipeout!
+  let stringified_events = map(s:logger.d_events,
+        \'v:val[0] . ": " . ' .
+        \'join(map(v:val[1], '.
+        \         '''substitute(v:val, '.
+        \                      '"\\v\\C(\\p@!.|\\<)", '.
+        \                      '"\\=printf(\"<%x>\", '.
+        \                                 'char2nr(submatch(0)))", '.
+        \                      '"")''), '.
+        \     '''\n'')')
+  call setline(1, [
+        \ 'Job exited with code ' . results[0],
+        \ printf('Screen (%u lines)', len(screen)),
+        \ repeat('=', 80),
+        \] +  screen + [
+        \ repeat('=', 80),
+        \ printf('Events (%u lines):', len(stringified_events)),
+        \ repeat('=', 80),
+        \] + stringified_events + [
+        \ repeat('=', 80),
+        \])
+  write
+  if results[0] != 0
+    if results[0] != -1
+      call jobstop(job)
+    endif
+    cquit
+  else
+    qall
+  endif
+endfunction
+
+call Main()