Merge branch 'master' into rename-execute

.travis.yml

@@ -43,11 +43,6 @@ env:
     # If this file exists, we know that the cache contains compiled
     # dependencies and we can use it.
     - CACHE_MARKER="$HOME/.cache/nvim-deps/.travis_cache_marker"
-    # Test success marker. If this file exists, we know that all tests
-    # were successful. Required because we only want to update the cache
-    # if the tests were successful, but don't have this information
-    # available in before_cache (which is run before after_success).
-    - SUCCESS_MARKER="$BUILD_DIR/.tests_successful"
     # default target name for functional tests
     - FUNCTIONALTEST=functionaltest
     - CI_TARGET=tests

CMakeLists.txt

@@ -275,7 +275,6 @@ else()
 endif()
 
 add_definitions(-DINCLUDE_GENERATED_DECLARATIONS)
-add_definitions(-DHAVE_CONFIG_H)
 
 if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")

ci/before_cache.sh

@@ -3,12 +3,15 @@
 set -e
 set -o pipefail
 
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/suite.sh"
+
 # Don't cache pip's log and selfcheck.
 rm -rf "${HOME}/.cache/pip/log"
 rm -f "${HOME}/.cache/pip/selfcheck.json"
 
 # Update the third-party dependency cache only if the build was successful.
-if [[ -f "${SUCCESS_MARKER}" ]]; then
+if ended_successfully; then
   rm -rf "${HOME}/.cache/nvim-deps"
   mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
   touch "${CACHE_MARKER}"

ci/common/suite.sh

@@ -2,11 +2,18 @@
 NL="$(printf '\nE')"
 NL="${NL%E}"
 
-FAILED=0
-
 FAIL_SUMMARY=""
 
+# Test success marker. If END_MARKER file exists, we know that all tests 
+# finished. If FAIL_SUMMARY_FILE exists we know that some tests failed, this 
+# file will contain information about failed tests. Build is considered 
+# successful if tests ended without any of them failing.
+END_MARKER="$BUILD_DIR/.tests_finished"
+FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors"
+
 enter_suite() {
+  FAILED=0
+  rm -f "${END_MARKER}"
   local suite_name="$1"
   export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
 }
@@ -19,17 +26,16 @@ exit_suite() {
   export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
   if test "x$1" != "x--continue" ; then
     exit $FAILED
+  else
+    local saved_failed=$FAILED
+    FAILED=0
+    return $saved_failed
   fi
 }
 
 fail() {
-  local allow_failure=
-  if test "x$1" = "x--allow-failure" ; then
-    shift
-    allow_failure=A
-  fi
   local test_name="$1"
-  local fail_char="$allow_failure$2"
+  local fail_char="$2"
   local message="$3"
 
   : ${fail_char:=F}
@@ -37,10 +43,9 @@ fail() {
 
   local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
   FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}"
   echo "Failed: $full_msg"
-  if test "x$allow_failure" = "x" ; then
+  FAILED=1
-    FAILED=1
-  fi
 }
 
 run_test() {
@@ -55,6 +60,12 @@ run_test() {
 }
 
 run_test_wd() {
+  local hang_ok=
+  if test "x$1" = "x--allow-hang" ; then
+    hang_ok=1
+    shift
+  fi
+
   local timeout="$1"
   test $# -gt 0 && shift
 
@@ -77,14 +88,13 @@ run_test_wd() {
   while test $restarts -gt 0 ; do
     : > "${status_file}"
     (
-      FAILED=0
+      set -o pipefail
-      if ! (
+      ret=0
-        set -o pipefail
+      if ! eval "$cmd" 2>&1 | tee -a "$output_file" ; then
-        eval "$cmd" 2>&1 | tee -a "$output_file"
+        ret=1
-      ) ; then
-        fail "${test_name}" "$@"
       fi
-      echo "$FAILED" > "$status_file"
+      echo "$ret" > "$status_file"
+      exit $ret
     ) &
     local pid=$!
     while test "$(stat -c "%s" "$status_file")" -eq 0 ; do
@@ -101,7 +111,9 @@ run_test_wd() {
       # status file not updated, assuming hang
       kill -KILL $pid
       if test $restarts -eq 0 ; then
-        fail "${test_name}" E "Test hang up"
+        if test "x$hang_ok" = "x" ; then
+          fail "${test_name}" E "Test hang up"
+        fi
       else
         echo "Test ${test_name} hang up, restarting"
         eval "$restart_cmd"
@@ -116,6 +128,20 @@ run_test_wd() {
   done
 }
 
-succeeded() {
+ended_successfully() {
-  return $FAILED
+  if [[ -f "${FAIL_SUMMARY_FILE}" ]]; then
+    echo 'Test failed, complete summary:'
+    cat "${FAIL_SUMMARY_FILE}"
+    return 1
+  fi
+  if ! [[ -f "${END_MARKER}" ]] ; then
+    echo 'ended_successfully called before end marker was touched'
+    return 1
+  fi
+  return 0
+}
+
+end_tests() {
+  touch "${END_MARKER}"
+  ended_successfully
 }

ci/common/test.sh

@@ -1,4 +1,5 @@
 source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/suite.sh"
 
 print_core() {
   local app="$1"
@@ -40,10 +41,9 @@ check_core_dumps() {
       print_core "$app" "$core"
     fi
   done
-  if test "$app" = quiet ; then
+  if test "$app" != quiet ; then
-    return 0
+    fail 'cores' E 'Core dumps found'
   fi
-  exit 1
 }
 
 check_logs() {
@@ -62,8 +62,7 @@ check_logs() {
     err=1
   done
   if [[ -n "${err}" ]]; then
-    echo "Runtime errors detected."
+    fail 'logs' E 'Runtime errors detected.'
-    exit 1
   fi
 }
 
@@ -75,50 +74,53 @@ asan_check() {
   check_logs "${1}" "*san.*"
 }
 
-run_unittests() {
+run_unittests() {(
+  enter_suite unittests
   ulimit -c unlimited
   if ! build_make unittest ; then
-    check_core_dumps "$(which luajit)"
+    fail 'unittests' F 'Unit tests failed'
-    exit 1
   fi
   check_core_dumps "$(which luajit)"
-}
+  exit_suite
+)}
 
-run_functionaltests() {
+run_functionaltests() {(
+  enter_suite functionaltests
   ulimit -c unlimited
   if ! build_make ${FUNCTIONALTEST}; then
-    asan_check "${LOG_DIR}"
+    fail 'functionaltests' F 'Functional tests failed'
-    valgrind_check "${LOG_DIR}"
-    check_core_dumps
-    exit 1
   fi
   asan_check "${LOG_DIR}"
   valgrind_check "${LOG_DIR}"
   check_core_dumps
-}
+  exit_suite
+)}
 
-run_oldtests() {
+run_oldtests() {(
+  enter_suite oldtests
   ulimit -c unlimited
   if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then
     reset
-    asan_check "${LOG_DIR}"
+    fail 'oldtests' F 'Legacy tests failed'
-    valgrind_check "${LOG_DIR}"
-    check_core_dumps
-    exit 1
   fi
   asan_check "${LOG_DIR}"
   valgrind_check "${LOG_DIR}"
   check_core_dumps
-}
+  exit_suite
+)}
 
-install_nvim() {
+install_nvim() {(
-  build_make install
+  enter_suite 'install_nvim'
+  if ! build_make install ; then
+    fail 'install' E 'make install failed'
+    exit_suite
+  fi
 
   "${INSTALL_PREFIX}/bin/nvim" --version
   "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
     echo "Running ':help' in the installed nvim failed."
     echo "Maybe the helptags have not been generated properly."
-    exit 1
+    fail 'help' F 'Failed running :help'
   }
 
   local genvimsynf=syntax/vim/generated.vim
@@ -127,24 +129,22 @@ install_nvim() {
     cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$'
   ) ; do
     if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
-      echo "It appears that $file is not installed."
+      fail 'runtime-install' F "It appears that $file is not installed."
-      exit 1
     fi
   done
 
   # Check that generated syntax file has function names, #5060.
   local gpat='syn keyword vimFuncName .*eval'
   if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then
-    echo "It appears that $genvimsynf does not contain $gpat."
+    fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat."
-    exit 1
   fi
 
   for file in $(
     cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$'
   ) ; do
     if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
-      echo "It appears that $file is not installed or is not executable."
+      fail 'not-exe' F "It appears that $file is not installed or is not executable."
-      exit 1
     fi
   done
-}
+  exit_suite
+)}

ci/run_lint.sh

@@ -20,9 +20,10 @@ csi_clean() {
 run_test 'top_make clint-full' clint
 run_test 'top_make testlint' testlint
 CLICOLOR_FORCE=1 run_test_wd \
-  5s \
+  --allow-hang \
+  10s \
   'top_make check-single-includes' \
   'csi_clean' \
   single-includes
 
-exit_suite
+end_tests

ci/run_tests.sh

@@ -27,8 +27,4 @@ run_test run_oldtests
 
 run_test install_nvim
 
-if succeeded ; then
+end_tests
-  touch "${SUCCESS_MARKER}"
-fi
-
-exit_suite

cmake/InstallClintErrors.cmake

@@ -0,0 +1,2 @@
+file(GLOB_RECURSE JSON_FILES *.json)
+file(COPY ${JSON_FILES} DESTINATION "${TARGET}")

runtime/doc/eval.txt

@@ -4944,8 +4944,8 @@ json_decode({expr})					*json_decode()*
 
 json_encode({expr})					*json_encode()*
 		Convert {expr} into a JSON string.  Accepts 
-		|msgpack-special-dict| as the input.  Will not convert |Funcref|s, 
+		|msgpack-special-dict| as the input.  Will not convert 
-		mappings with non-string keys (can be created as 
+		|Funcref|s, mappings with non-string keys (can be created as 
 		|msgpack-special-dict|), values with self-referencing 
 		containers, strings which contain non-UTF-8 characters, 
 		pseudo-UTF-8 strings which contain codepoints reserved for 

runtime/doc/options.txt

@@ -5151,8 +5151,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 		saved.  When not included, the value of 'history' is used.
 							*shada-c*
 	c	Dummy option, kept for compatibility reasons.  Has no actual 
-		effect.  Current encoding state is described in 
+		effect: ShaDa always uses UTF-8 and 'encoding' value is fixed 
-		|shada-encoding|.
+		to UTF-8 as well.
 							*shada-f*
 	f	Whether file marks need to be stored.  If zero, file marks ('0
 		to '9, 'A to 'Z) are not stored.  When not present or when

runtime/doc/starting.txt

@@ -1097,23 +1097,6 @@ SHADA FILE NAME						*shada-file-name*
   default and the name given with 'shada' or "-i" (unless it's NONE).
 
 
-CHARACTER ENCODING					*shada-encoding*
-
-The text in the ShaDa file is UTF-8-encoded.  Normally you will always work 
-with the same 'encoding' value, and this works just fine.  However, if you 
-read the ShaDa file with value for 'encoding' different from utf-8 and 
-'encoding' used when writing ShaDa file, some of the text (non-ASCII 
-characters) may be invalid as Neovim always attempts to convert the text in 
-the ShaDa file from the UTF-8 to the current 'encoding' value.  Filenames are 
-never converted, affected elements are:
-
-- history strings;
-- variable values;
-- register values;
-- last used search and substitute patterns;
-- last used substitute replacement string.
-
-
 MANUALLY READING AND WRITING				*shada-read-write*
 
 Two commands can be used to read and write the ShaDa file manually.  This
@@ -1221,8 +1204,11 @@ exactly four MessagePack objects:
 3. Third goes the length of the fourth entry.  Unsigned integer as well, used 
    for fast skipping without parsing.
 4. Fourth is actual entry data.  All currently used ShaDa entries use 
-   containers to hold data: either map or array.  Exact format depends on the 
+   containers to hold data: either map or array.  All string values in those 
-   entry type:
+   containers are either binary (applies to filenames) or UTF-8, yet parser 
+   needs to expect that invalid bytes may be present in a UTF-8 string.
+
+   Exact format depends on the entry type:
 
    Entry type (name)   Entry data ~
    1 (Header)          Map containing data that describes the generator 

src/nvim/CMakeLists.txt

@@ -41,7 +41,10 @@ set(LINT_SUPPRESS_URL "${LINT_SUPPRESS_URL_BASE}/errors.json")
 set(LINT_PRG ${PROJECT_SOURCE_DIR}/src/clint.py)
 set(DOWNLOAD_SCRIPT ${PROJECT_SOURCE_DIR}/cmake/Download.cmake)
 set(LINT_SUPPRESSES_ROOT ${PROJECT_BINARY_DIR}/errors)
-set(LINT_SUPPRESSES_URL "https://raw.githubusercontent.com/neovim/doc/gh-pages/reports/clint/errors.tar.gz")
+set(LINT_SUPPRESSES_URL "${LINT_SUPPRESS_URL_BASE}/errors.tar.gz")
+set(LINT_SUPPRESSES_ARCHIVE ${LINT_SUPPRESSES_ROOT}/errors.tar.gz)
+set(LINT_SUPPRESSES_TOUCH_FILE "${TOUCHES_DIR}/unpacked-clint-errors-archive")
+set(LINT_SUPPRESSES_INSTALL_SCRIPT "${PROJECT_SOURCE_DIR}/cmake/InstallClintErrors.cmake")
 
 include_directories(${GENERATED_DIR})
 include_directories(${CACHED_GENERATED_DIR})
@@ -50,6 +53,8 @@ include_directories(${GENERATED_INCLUDES_DIR})
 file(MAKE_DIRECTORY ${TOUCHES_DIR})
 file(MAKE_DIRECTORY ${GENERATED_DIR})
 file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR})
+file(MAKE_DIRECTORY ${LINT_SUPPRESSES_ROOT})
+file(MAKE_DIRECTORY ${LINT_SUPPRESSES_ROOT}/src)
 
 file(GLOB NVIM_SOURCES *.c)
 file(GLOB NVIM_HEADERS *.h)
@@ -486,6 +491,20 @@ function(add_download output url allow_failure)
   )
 endfunction()
 
+add_download(${LINT_SUPPRESSES_ARCHIVE} ${LINT_SUPPRESSES_URL} off)
+
+add_custom_command(
+  OUTPUT ${LINT_SUPPRESSES_TOUCH_FILE}
+  WORKING_DIRECTORY ${LINT_SUPPRESSES_ROOT}/src
+  COMMAND ${CMAKE_COMMAND} -E tar xfz ${LINT_SUPPRESSES_ARCHIVE}
+  COMMAND
+    ${CMAKE_COMMAND}
+      -DTARGET=${LINT_SUPPRESSES_ROOT}
+      -P ${LINT_SUPPRESSES_INSTALL_SCRIPT}
+  DEPENDS
+    ${LINT_SUPPRESSES_ARCHIVE} ${LINT_SUPPRESSES_INSTALL_SCRIPT}
+)
+
 add_download(${LINT_SUPPRESS_FILE} ${LINT_SUPPRESS_URL} off)
 
 set(LINT_NVIM_REL_SOURCES)
@@ -494,14 +513,13 @@ foreach(sfile ${LINT_NVIM_SOURCES})
   set(suppress_file ${LINT_SUPPRESSES_ROOT}/${suffix}.json)
   set(suppress_url "${LINT_SUPPRESS_URL_BASE}/${suffix}.json")
   set(rsfile src/nvim/${r})
-  add_download(${suppress_file} ${suppress_url} on)
   set(touch_file "${TOUCHES_DIR}/ran-clint-${suffix}")
   add_custom_command(
     OUTPUT ${touch_file}
     COMMAND ${LINT_PRG} --suppress-errors=${suppress_file} ${rsfile}
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     COMMAND ${CMAKE_COMMAND} -E touch ${touch_file}
-    DEPENDS ${LINT_PRG} ${sfile} ${suppress_file}
+    DEPENDS ${LINT_PRG} ${sfile} ${LINT_SUPPRESSES_TOUCH_FILE}
   )
   list(APPEND LINT_TARGETS ${touch_file})
   list(APPEND LINT_NVIM_REL_SOURCES ${rsfile})

src/nvim/digraph.h

@@ -1,8 +1,8 @@
 #ifndef NVIM_DIGRAPH_H
 #define NVIM_DIGRAPH_H
 
-#include "nvim/types.h" // for char_u
+#include "nvim/types.h"
-#include "nvim/ex_cmds_defs.h" // for exarg_T
+#include "nvim/ex_cmds_defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "digraph.h.generated.h"

src/nvim/eval/decode.c

@@ -222,8 +222,6 @@ static inline int json_decoder_pop(ValuesStackItem obj,
 
 /// Parse JSON double-quoted string
 ///
-/// @param[in]  conv  Defines conversion necessary to convert UTF-8 string to
-///                   &encoding.
 /// @param[in]  buf  Buffer being converted.
 /// @param[in]  buf_len  Length of the buffer.
 /// @param[in,out]  pp  Pointer to the start of the string. Must point to '"'.
@@ -240,8 +238,7 @@ static inline int json_decoder_pop(ValuesStackItem obj,
 ///                        value when decoder is restarted, otherwise unused.
 ///
 /// @return OK in case of success, FAIL in case of error.
-static inline int parse_json_string(vimconv_T *const conv,
+static inline int parse_json_string(const char *const buf, const size_t buf_len,
-                                    const char *const buf, const size_t buf_len,
                                     const char **const pp,
                                     ValuesStack *const stack,
                                     ContainerStack *const container_stack,
@@ -416,20 +413,6 @@ static inline int parse_json_string(vimconv_T *const conv,
   }
   PUT_FST_IN_PAIR(fst_in_pair, str_end);
 #undef PUT_FST_IN_PAIR
-  if (conv->vc_type != CONV_NONE) {
-    size_t str_len = (size_t) (str_end - str);
-    char *const new_str = (char *) string_convert(conv, (char_u *) str,
-                                                  &str_len);
-    if (new_str == NULL) {
-      emsgf(_("E474: Failed to convert string \"%.*s\" from UTF-8"),
-            (int) str_len, str);
-      xfree(str);
-      goto parse_json_string_fail;
-    }
-    xfree(str);
-    str = new_str;
-    str_end = new_str + str_len;
-  }
   if (hasnul) {
     typval_T obj;
     list_T *const list = tv_list_alloc();
@@ -626,9 +609,6 @@ int json_decode_string(const char *const buf, const size_t buf_len,
     EMSG(_("E474: Attempt to decode a blank string"));
     return FAIL;
   }
-  vimconv_T conv = { .vc_type = CONV_NONE };
-  convert_setup(&conv, (char_u *) "utf-8", p_enc);
-  conv.vc_fail = true;
   int ret = OK;
   ValuesStack stack = KV_INITIAL_VALUE;
   ContainerStack container_stack = KV_INITIAL_VALUE;
@@ -774,7 +754,7 @@ json_decode_string_cycle_start:
         break;
       }
       case '"': {
-        if (parse_json_string(&conv, buf, buf_len, &p, &stack, &container_stack,
+        if (parse_json_string(buf, buf_len, &p, &stack, &container_stack,
                               &next_map_special, &didcomma, &didcolon)
             == FAIL) {
           // Error message was already given

src/nvim/eval/encode.c

@@ -11,7 +11,7 @@
 #include <math.h>
 
 #include "nvim/eval/encode.h"
-#include "nvim/buffer_defs.h"  // vimconv_T
+#include "nvim/buffer_defs.h"
 #include "nvim/eval.h"
 #include "nvim/eval/typval.h"
 #include "nvim/garray.h"
@@ -29,10 +29,6 @@
 #define utf_ptr2char(b) utf_ptr2char((char_u *)b)
 #define utf_ptr2len(b) ((size_t)utf_ptr2len((char_u *)b))
 #define utf_char2len(b) ((size_t)utf_char2len(b))
-#define string_convert(a, b, c) \
-      ((char *)string_convert((vimconv_T *)a, (char_u *)b, c))
-#define convert_setup(vcp, from, to) \
-    (convert_setup(vcp, (char_u *)from, (char_u *)to))
 
 const char *const encode_special_var_names[] = {
   [kSpecialVarNull] = "null",
@@ -537,17 +533,6 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
       } \
     } while (0)
 
-/// Last used p_enc value
-///
-/// Generic pointer: it is not used as a string, only pointer comparisons are
-/// performed. Must not be freed.
-static const void *last_p_enc = NULL;
-
-/// Conversion setup for converting from last_p_enc to UTF-8
-static vimconv_T p_enc_conv = {
-  .vc_type = CONV_NONE,
-};
-
 /// Escape sequences used in JSON
 static const char escapes[][3] = {
   [BS] = "\\b",
@@ -579,33 +564,15 @@ static inline int convert_to_json_string(garray_T *const gap,
   } else {
     size_t utf_len = len;
     char *tofree = NULL;
-    if (last_p_enc != (const void *) p_enc) {
-      p_enc_conv.vc_type = CONV_NONE;
-      convert_setup(&p_enc_conv, p_enc, "utf-8");
-      p_enc_conv.vc_fail = true;
-      last_p_enc = p_enc;
-    }
-    if (p_enc_conv.vc_type != CONV_NONE) {
-      tofree = string_convert(&p_enc_conv, buf, &utf_len);
-      if (tofree == NULL) {
-        emsgf(_("E474: Failed to convert string \"%.*s\" to UTF-8"),
-              utf_len, utf_buf);
-        return FAIL;
-      }
-      utf_buf = tofree;
-    }
     size_t str_len = 0;
-    // Encode character as \u0000 if
+    // Encode character as \uNNNN if
-    // 1. It is an ASCII control character (0x0 .. 0x1F, 0x7F).
+    // 1. It is an ASCII control character (0x0 .. 0x1F; 0x7F not
-    // 2. &encoding is not UTF-8 and code point is above 0x7F.
+    //    utf_printable and thus not checked specially).
-    // 3. &encoding is UTF-8 and code point is not printable according to
+    // 2. Code point is not printable according to utf_printable().
-    //    utf_printable().
+    // This is done to make resulting values displayable on screen also not from
-    // This is done to make it possible to :echo values when &encoding is not
+    // Neovim.
-    // UTF-8.
+#define ENCODE_RAW(ch) \
-#define ENCODE_RAW(p_enc_conv, ch) \
+    (ch >= 0x20 && utf_printable(ch))
-    (ch >= 0x20 && (p_enc_conv.vc_type == CONV_NONE \
-                    ? utf_printable(ch) \
-                    : ch < 0x7F))
     for (size_t i = 0; i < utf_len;) {
       const int ch = utf_ptr2char(utf_buf + i);
       const size_t shift = (ch == 0? 1: utf_ptr2len(utf_buf + i));
@@ -636,7 +603,7 @@ static inline int convert_to_json_string(garray_T *const gap,
                   utf_len - (i - shift), utf_buf + i - shift);
             xfree(tofree);
             return FAIL;
-          } else if (ENCODE_RAW(p_enc_conv, ch)) {
+          } else if (ENCODE_RAW(ch)) {
             str_len += shift;
           } else {
             str_len += ((sizeof("\\u1234") - 1)
@@ -666,7 +633,7 @@ static inline int convert_to_json_string(garray_T *const gap,
           break;
         }
         default: {
-          if (ENCODE_RAW(p_enc_conv, ch)) {
+          if (ENCODE_RAW(ch)) {
             ga_concat_len(gap, utf_buf + i, shift);
           } else if (ch < SURROGATE_FIRST_CHAR) {
             ga_concat_len(gap, ((const char[]) {

src/nvim/ex_cmds.h

@@ -6,8 +6,8 @@
 #include "nvim/os/time.h"
 #include "nvim/pos.h"
 #include "nvim/eval/typval.h"
-#include "nvim/buffer_defs.h" // for buf_T and win_T
+#include "nvim/buffer_defs.h"
-#include "nvim/ex_cmds_defs.h" // for exarg_T
+#include "nvim/ex_cmds_defs.h"
 
 // flags for do_ecmd()
 #define ECMD_HIDE            0x01    // don't free the current buffer

src/nvim/ex_getln.h

@@ -3,9 +3,9 @@
 
 #include "nvim/eval/typval.h"
 #include "nvim/ex_cmds.h"
-#include "nvim/ex_cmds_defs.h" // for exarg_T
+#include "nvim/ex_cmds_defs.h"
-#include "nvim/os/time.h" // for Timestamp
+#include "nvim/os/time.h"
-#include "nvim/regexp_defs.h" // for regmatch_T
+#include "nvim/regexp_defs.h"
 
 /* Values for nextwild() and ExpandOne().  See ExpandOne() for meaning. */
 #define WILD_FREE               1

src/nvim/fold.h

@@ -1,12 +1,12 @@
 #ifndef NVIM_FOLD_H
 #define NVIM_FOLD_H
 
-#include <stdio.h> // for FILE
+#include <stdio.h>
 
 #include "nvim/pos.h"
-#include "nvim/garray.h" // for garray_T
+#include "nvim/garray.h"
-#include "nvim/types.h" // for char_u
+#include "nvim/types.h"
-#include "nvim/buffer_defs.h" // for win_T
+#include "nvim/buffer_defs.h"
 
 /*
  * Info used to pass info about a fold from the fold-detection code to the

src/nvim/iconv.h

@@ -10,9 +10,7 @@
 // USE_ICONV, or to put the USE_ICONV definition in config.h.in directly. As
 // it stands, globals.h needs to be included alongside iconv.h.
 
-#ifdef HAVE_CONFIG_H
+#include "auto/config.h"
-# include "auto/config.h"
-#endif
 
 // Use iconv() when it's available, either by linking to the library at
 // compile time or by loading it at runtime.

src/nvim/memfile_defs.h

@@ -3,10 +3,10 @@
 
 #include <stdint.h>
 #include <stdbool.h>
-#include <stdlib.h> // for size_t
+#include <stdlib.h>
 
 #include "nvim/types.h"
-#include "nvim/pos.h" // for linenr_T
+#include "nvim/pos.h"
 
 /// A block number.
 ///

src/nvim/move.h

@@ -2,8 +2,8 @@
 #define NVIM_MOVE_H
 
 #include <stdbool.h>
-#include "nvim/buffer_defs.h" // for win_T
+#include "nvim/buffer_defs.h"
-#include "nvim/pos.h" // for linenr_T
+#include "nvim/pos.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "move.h.generated.h"

src/nvim/shada.c

@@ -73,15 +73,10 @@ KHASH_SET_INIT_STR(strset)
     (vim_rename((char_u *)a, (char_u *)b))
 #define mb_strnicmp(a, b, c) \
     (mb_strnicmp((char_u *)a, (char_u *)b, c))
-#define has_non_ascii(a) (has_non_ascii((char_u *)a))
-#define string_convert(a, b, c) \
-      ((char *)string_convert((vimconv_T *)a, (char_u *)b, c))
 #define path_shorten_fname_if_possible(b) \
     ((char *)path_shorten_fname_if_possible((char_u *)b))
 #define buflist_new(ffname, sfname, ...) \
     (buflist_new((char_u *)ffname, (char_u *)sfname, __VA_ARGS__))
-#define convert_setup(vcp, from, to) \
-    (convert_setup(vcp, (char_u *)from, (char_u *)to))
 #define os_isdir(f) (os_isdir((char_u *) f))
 #define regtilde(s, m) ((char *) regtilde((char_u *) s, m))
 #define path_tail_with_sep(f) ((char *) path_tail_with_sep((char_u *)f))
@@ -413,8 +408,6 @@ typedef struct sd_read_def {
   const char *error;      ///< Error message in case of error.
   uintmax_t fpos;         ///< Current position (amount of bytes read since
                           ///< reader structure initialization). May overflow.
-  vimconv_T sd_conv;      ///< Structure used for converting encodings of some
-                          ///< items.
 } ShaDaReadDef;
 
 struct sd_write_def;
@@ -435,8 +428,6 @@ typedef struct sd_write_def {
   ShaDaWriteCloser close;  ///< Close function.
   void *cookie;            ///< Data describing object written to.
   const char *error;       ///< Error message in case of error.
-  vimconv_T sd_conv;       ///< Structure used for converting encodings of some
-                           ///< items.
 } ShaDaWriteDef;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
@@ -803,7 +794,7 @@ static int open_shada_file_for_reading(const char *const fname,
     return error;
   }
 
-  convert_setup(&sd_reader->sd_conv, "utf-8", p_enc);
+  assert(STRCMP(p_enc, "utf-8") == 0);
 
   return 0;
 }
@@ -1899,127 +1890,24 @@ shada_pack_entry_error:
 }
 #undef PACK_STRING
 
-/// Write single ShaDa entry, converting it if needed
+/// Write single ShaDa entry and free it afterwards
 ///
-/// @warning Frees entry after packing.
+/// Will not free if entry could not be freed.
 ///
 /// @param[in]  packer     Packer used to write entry.
-/// @param[in]  sd_conv    Conversion definitions.
+/// @param[in]  entry      Entry written.
-/// @param[in]  entry      Entry written. If entry.can_free_entry is false then
-///                        it assumes that entry was not converted, otherwise it
-///                        is assumed that entry was already converted.
 /// @param[in]  max_kbyte  Maximum size of an item in KiB. Zero means no
 ///                        restrictions.
-static ShaDaWriteResult shada_pack_encoded_entry(msgpack_packer *const packer,
+static inline ShaDaWriteResult shada_pack_pfreed_entry(
-                                                 const vimconv_T *const sd_conv,
+    msgpack_packer *const packer, PossiblyFreedShadaEntry entry,
-                                                 PossiblyFreedShadaEntry entry,
+    const size_t max_kbyte)
-                                                 const size_t max_kbyte)
+  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE
-  FUNC_ATTR_NONNULL_ALL
 {
   ShaDaWriteResult ret = kSDWriteSuccessfull;
+  ret = shada_pack_entry(packer, entry.data, max_kbyte);
   if (entry.can_free_entry) {
-    ret = shada_pack_entry(packer, entry.data, max_kbyte);
     shada_free_shada_entry(&entry.data);
-    return ret;
   }
-#define RUN_WITH_CONVERTED_STRING(cstr, code) \
-  do { \
-    bool did_convert = false; \
-    if (sd_conv->vc_type != CONV_NONE && has_non_ascii((cstr))) { \
-      char *const converted_string = string_convert(sd_conv, (cstr), NULL); \
-      if (converted_string != NULL) { \
-        (cstr) = converted_string; \
-        did_convert = true; \
-      } \
-    } \
-    code \
-    if (did_convert) { \
-      xfree((cstr)); \
-    } \
-  } while (0)
-  switch (entry.data.type) {
-    case kSDItemUnknown:
-    case kSDItemMissing: {
-      assert(false);
-    }
-    case kSDItemSearchPattern: {
-      RUN_WITH_CONVERTED_STRING(entry.data.data.search_pattern.pat, {
-        ret = shada_pack_entry(packer, entry.data, max_kbyte);
-      });
-      break;
-    }
-    case kSDItemHistoryEntry: {
-      RUN_WITH_CONVERTED_STRING(entry.data.data.history_item.string, {
-        ret = shada_pack_entry(packer, entry.data, max_kbyte);
-      });
-      break;
-    }
-    case kSDItemSubString: {
-      RUN_WITH_CONVERTED_STRING(entry.data.data.sub_string.sub, {
-        ret = shada_pack_entry(packer, entry.data, max_kbyte);
-      });
-      break;
-    }
-    case kSDItemVariable: {
-      if (sd_conv->vc_type != CONV_NONE) {
-        typval_T tgttv;
-        var_item_copy(sd_conv, &entry.data.data.global_var.value, &tgttv,
-                      true, 0);
-        tv_clear(&entry.data.data.global_var.value);
-        entry.data.data.global_var.value = tgttv;
-      }
-      ret = shada_pack_entry(packer, entry.data, max_kbyte);
-      break;
-    }
-    case kSDItemRegister: {
-      bool did_convert = false;
-      if (sd_conv->vc_type != CONV_NONE) {
-        size_t first_non_ascii = 0;
-        for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
-          if (has_non_ascii(entry.data.data.reg.contents[i])) {
-            first_non_ascii = i;
-            did_convert = true;
-            break;
-          }
-        }
-        if (did_convert) {
-          entry.data.data.reg.contents =
-              xmemdup(entry.data.data.reg.contents,
-                      (entry.data.data.reg.contents_size
-                       * sizeof(entry.data.data.reg.contents[0])));
-          for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
-            if (i >= first_non_ascii) {
-              entry.data.data.reg.contents[i] = get_converted_string(
-                  sd_conv,
-                  entry.data.data.reg.contents[i],
-                  strlen(entry.data.data.reg.contents[i]));
-            } else {
-              entry.data.data.reg.contents[i] =
-                  xstrdup(entry.data.data.reg.contents[i]);
-            }
-          }
-        }
-      }
-      ret = shada_pack_entry(packer, entry.data, max_kbyte);
-      if (did_convert) {
-        for (size_t i = 0; i < entry.data.data.reg.contents_size; i++) {
-          xfree(entry.data.data.reg.contents[i]);
-        }
-        xfree(entry.data.data.reg.contents);
-      }
-      break;
-    }
-    case kSDItemHeader:
-    case kSDItemGlobalMark:
-    case kSDItemJump:
-    case kSDItemBufferList:
-    case kSDItemLocalMark:
-    case kSDItemChange: {
-      ret = shada_pack_entry(packer, entry.data, max_kbyte);
-      break;
-    }
-  }
-#undef RUN_WITH_CONVERTED_STRING
   return ret;
 }
 
@@ -2556,11 +2444,7 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
         break;
       }
       typval_T tgttv;
-      if (sd_writer->sd_conv.vc_type != CONV_NONE) {
+      tv_copy(&vartv, &tgttv);
-        var_item_copy(&sd_writer->sd_conv, &vartv, &tgttv, true, 0);
-      } else {
-        tv_copy(&vartv, &tgttv);
-      }
       ShaDaWriteResult spe_ret;
       if ((spe_ret = shada_pack_entry(packer, (ShadaEntry) {
         .type = kSDItemVariable,
@@ -2811,9 +2695,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
   do { \
     for (size_t i_ = 0; i_ < ARRAY_SIZE(wms_array); i_++) { \
       if (wms_array[i_].data.type != kSDItemMissing) { \
-        if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, \
+        if (shada_pack_pfreed_entry(packer, wms_array[i_], max_kbyte) \
-                                     wms_array[i_], \
+            == kSDWriteFailed) { \
-                                     max_kbyte) == kSDWriteFailed) { \
           ret = kSDWriteFailed; \
           goto shada_write_exit; \
         } \
@@ -2823,8 +2706,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
   PACK_WMS_ARRAY(wms->global_marks);
   PACK_WMS_ARRAY(wms->registers);
   for (size_t i = 0; i < wms->jumps_size; i++) {
-    if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms->jumps[i],
+    if (shada_pack_pfreed_entry(packer, wms->jumps[i], max_kbyte)
-                                 max_kbyte) == kSDWriteFailed) {
+        == kSDWriteFailed) {
       ret = kSDWriteFailed;
       goto shada_write_exit;
     }
@@ -2832,8 +2715,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
 #define PACK_WMS_ENTRY(wms_entry) \
   do { \
     if (wms_entry.data.type != kSDItemMissing) { \
-      if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv, wms_entry, \
+      if (shada_pack_pfreed_entry(packer, wms_entry, max_kbyte) \
-                                   max_kbyte) == kSDWriteFailed) { \
+          == kSDWriteFailed) { \
         ret = kSDWriteFailed; \
         goto shada_write_exit; \
       } \
@@ -2860,9 +2743,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
   for (size_t i = 0; i < file_markss_to_dump; i++) {
     PACK_WMS_ARRAY(all_file_markss[i]->marks);
     for (size_t j = 0; j < all_file_markss[i]->changes_size; j++) {
-      if (shada_pack_encoded_entry(packer, &sd_writer->sd_conv,
+      if (shada_pack_pfreed_entry(packer, all_file_markss[i]->changes[j],
-                                   all_file_markss[i]->changes[j],
+                                  max_kbyte) == kSDWriteFailed) {
-                                   max_kbyte) == kSDWriteFailed) {
         ret = kSDWriteFailed;
         goto shada_write_exit;
       }
@@ -2886,8 +2768,8 @@ static ShaDaWriteResult shada_write(ShaDaWriteDef *const sd_writer,
       if (dump_one_history[i]) {
         hms_insert_whole_neovim_history(&wms->hms[i]);
         HMS_ITER(&wms->hms[i], cur_entry, {
-          if (shada_pack_encoded_entry(
+          if (shada_pack_pfreed_entry(
-              packer, &sd_writer->sd_conv, (PossiblyFreedShadaEntry) {
+              packer, (PossiblyFreedShadaEntry) {
                 .data = cur_entry->data,
                 .can_free_entry = cur_entry->can_free_entry,
               }, max_kbyte) == kSDWriteFailed) {
@@ -3038,8 +2920,6 @@ shada_write_file_nomerge: {}
     verbose_leave();
   }
 
-  convert_setup(&sd_writer.sd_conv, p_enc, "utf-8");
-
   const ShaDaWriteResult sw_ret = shada_write(&sd_writer, (nomerge
                                                            ? NULL
                                                            : &sd_reader));
@@ -3327,29 +3207,6 @@ static ShaDaReadResult msgpack_read_uint64(ShaDaReadDef *const sd_reader,
   return kSDReadStatusSuccess;
 }
 
-/// Convert or copy and return a string
-///
-/// @param[in]  sd_conv  Conversion definition.
-/// @param[in]  str      String to convert.
-/// @param[in]  len      String length.
-///
-/// @return [allocated] converted string or copy of the original string.
-static inline char *get_converted_string(const vimconv_T *const sd_conv,
-                                         const char *const str,
-                                         const size_t len)
-  FUNC_ATTR_NONNULL_ALL FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT
-{
-  if (!has_non_ascii_len(str, len)) {
-    return xmemdupz(str, len);
-  }
-  size_t new_len = len;
-  char *const new_str = string_convert(sd_conv, str, &new_len);
-  if (new_str == NULL) {
-    return xmemdupz(str, len);
-  }
-  return new_str;
-}
-
 #define READERR(entry_name, error_desc) \
     RERR "Error while reading ShaDa file: " \
     entry_name " entry at position %" PRIu64 " " \
@@ -3427,10 +3284,7 @@ static inline char *get_converted_string(const vimconv_T *const sd_conv,
            sizeof(*unpacked.data.via.map.ptr)); \
     ad_ga.ga_len++; \
   }
-#define CONVERTED(str, len) ( \
+#define CONVERTED(str, len) (xmemdupz((str), (len)))
-    sd_reader->sd_conv.vc_type != CONV_NONE \
-    ? get_converted_string(&sd_reader->sd_conv, (str), (len)) \
-    : xmemdupz((str), (len)))
 #define BIN_CONVERTED(b) CONVERTED(b.ptr, b.size)
 #define SET_ADDITIONAL_DATA(tgt, name) \
       do { \
@@ -3803,30 +3657,14 @@ shada_read_next_item_start:
             (char) unpacked.data.via.array.ptr[2].via.u64;
       }
       size_t strsize;
-      if (sd_reader->sd_conv.vc_type == CONV_NONE
+      strsize = (
-          || !has_non_ascii_len(unpacked.data.via.array.ptr[1].via.bin.ptr,
+          unpacked.data.via.array.ptr[1].via.bin.size
-                                unpacked.data.via.array.ptr[1].via.bin.size)) {
+          + 1  // Zero byte
-shada_read_next_item_hist_no_conv:
+          + 1);  // Separator character
-        strsize = (
+      entry->data.history_item.string = xmalloc(strsize);
-            unpacked.data.via.array.ptr[1].via.bin.size
+      memcpy(entry->data.history_item.string,
-            + 1  // Zero byte
+             unpacked.data.via.array.ptr[1].via.bin.ptr,
-            + 1);  // Separator character
+             unpacked.data.via.array.ptr[1].via.bin.size);
-        entry->data.history_item.string = xmalloc(strsize);
-        memcpy(entry->data.history_item.string,
-               unpacked.data.via.array.ptr[1].via.bin.ptr,
-               unpacked.data.via.array.ptr[1].via.bin.size);
-      } else {
-        size_t len = unpacked.data.via.array.ptr[1].via.bin.size;
-        char *const converted = string_convert(
-            &sd_reader->sd_conv, unpacked.data.via.array.ptr[1].via.bin.ptr,
-            &len);
-        if (converted != NULL) {
-          strsize = len + 2;
-          entry->data.history_item.string = xrealloc(converted, strsize);
-        } else {
-          goto shada_read_next_item_hist_no_conv;
-        }
-      }
       entry->data.history_item.string[strsize - 2] = 0;
       entry->data.history_item.string[strsize - 1] =
           entry->data.history_item.sep;
@@ -3859,16 +3697,6 @@ shada_read_next_item_hist_no_conv:
                         "be converted to the VimL value")), initial_fpos);
         goto shada_read_next_item_error;
       }
-      if (sd_reader->sd_conv.vc_type != CONV_NONE) {
-        typval_T tgttv;
-        var_item_copy(&sd_reader->sd_conv,
-                      &entry->data.global_var.value,
-                      &tgttv,
-                      true,
-                      0);
-        tv_clear(&entry->data.global_var.value);
-        entry->data.global_var.value = tgttv;
-      }
       SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2,
                               entry->data.global_var.additional_elements,
                               "variable");

src/nvim/vim.h

@@ -11,20 +11,16 @@
 #define RUNTIME_DIRNAME "runtime"
 /* end */
 
-/* ============ the header file puzzle (ca. 50-100 pieces) ========= */
+#include "auto/config.h"
-
+#define HAVE_PATHDEF
-#ifdef HAVE_CONFIG_H    /* GNU autoconf (or something else) was here */
-# include "auto/config.h"
-# define HAVE_PATHDEF
 
 /*
  * Check if configure correctly managed to find sizeof(int).  If this failed,
  * it becomes zero.  This is likely a problem of not being able to run the
  * test program.  Other items from configure may also be wrong then!
  */
-# if (SIZEOF_INT == 0)
+#if (SIZEOF_INT == 0)
-Error: configure did not run properly.Check auto/config.log.
+# error Configure did not run properly.
-# endif
 #endif
 
 #include "nvim/os/os_defs.h"       /* bring lots of system header files */
@@ -46,11 +42,6 @@ enum { NUMBUFLEN = 65 };
 #include "nvim/keymap.h"
 #include "nvim/macros.h"
 
-
-
-
-/* ================ end of the header file puzzle =============== */
-
 #include "nvim/gettext.h"
 
 /* special attribute addition: Put message in history */

test/functional/helpers.lua

@@ -251,12 +251,13 @@ local function retry(max, max_ms, fn)
       return result
     end
     if (max and tries >= max) or (luv.now() - start_time > timeout) then
-      break
+      if type(result) == "string" then
+        result = "\nretry() attempts: "..tostring(tries).."\n"..result
+      end
+      error(result)
     end
     tries = tries + 1
   end
-  -- Do not use pcall() for the final attempt, let the failure bubble up.
-  return fn()
 end
 
 local function clear(...)

test/functional/terminal/tui_spec.lua

@@ -2,9 +2,10 @@
 -- as a simple way to send keys and assert screen state.
 local helpers = require('test.functional.helpers')(after_each)
 local thelpers = require('test.functional.terminal.helpers')
-local feed = thelpers.feed_data
+local feed_data = thelpers.feed_data
 local feed_command = helpers.feed_command
 local nvim_dir = helpers.nvim_dir
+local retry = helpers.retry
 
 if helpers.pending_win32(pending) then return end
 
@@ -34,7 +35,7 @@ describe('tui', function()
   end)
 
   it('accepts basic utf-8 input', function()
-    feed('iabc\ntest1\ntest2')
+    feed_data('iabc\ntest1\ntest2')
     screen:expect([[
       abc                                               |
       test1                                             |
@@ -44,7 +45,7 @@ describe('tui', function()
       {3:-- INSERT --}                                      |
       {3:-- TERMINAL --}                                    |
     ]])
-    feed('\027')
+    feed_data('\027')
     screen:expect([[
       abc                                               |
       test1                                             |
@@ -60,7 +61,7 @@ describe('tui', function()
     local keys = 'dfghjkl'
     for c in keys:gmatch('.') do
       feed_command('nnoremap <a-'..c..'> ialt-'..c..'<cr><esc>')
-      feed('\027'..c)
+      feed_data('\027'..c)
     end
     screen:expect([[
       alt-j                                             |
@@ -71,7 +72,7 @@ describe('tui', function()
                                                         |
       {3:-- TERMINAL --}                                    |
     ]])
-    feed('gg')
+    feed_data('gg')
     screen:expect([[
       {1:a}lt-d                                             |
       alt-f                                             |
@@ -90,7 +91,7 @@ describe('tui', function()
     -- Example: for input ALT+j:
     --    * Vim (Nvim prior to #3982) sets high-bit, inserts "ê".
     --    * Nvim (after #3982) inserts "j".
-    feed('i\027j')
+    feed_data('i\027j')
     screen:expect([[
       j{1: }                                                |
       {4:~                                                 }|
@@ -103,10 +104,10 @@ describe('tui', function()
   end)
 
   it('accepts ascii control sequences', function()
-    feed('i')
+    feed_data('i')
-    feed('\022\007') -- ctrl+g
+    feed_data('\022\007') -- ctrl+g
-    feed('\022\022') -- ctrl+v
+    feed_data('\022\022') -- ctrl+v
-    feed('\022\013') -- ctrl+m
+    feed_data('\022\013') -- ctrl+m
     screen:expect([[
     {9:^G^V^M}{1: }                                           |
     {4:~                                                 }|
@@ -119,7 +120,7 @@ describe('tui', function()
   end)
 
   it('automatically sends <Paste> for bracketed paste sequences', function()
-    feed('i\027[200~')
+    feed_data('i\027[200~')
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -129,7 +130,7 @@ describe('tui', function()
       {3:-- INSERT (paste) --}                              |
       {3:-- TERMINAL --}                                    |
     ]])
-    feed('pasted from terminal')
+    feed_data('pasted from terminal')
     screen:expect([[
       pasted from terminal{1: }                             |
       {4:~                                                 }|
@@ -139,7 +140,7 @@ describe('tui', function()
       {3:-- INSERT (paste) --}                              |
       {3:-- TERMINAL --}                                    |
     ]])
-    feed('\027[201~')
+    feed_data('\027[201~')
     screen:expect([[
       pasted from terminal{1: }                             |
       {4:~                                                 }|
@@ -157,7 +158,7 @@ describe('tui', function()
     for i = 1, 3000 do
       t[i] = 'item ' .. tostring(i)
     end
-    feed('i\027[200~'..table.concat(t, '\n')..'\027[201~')
+    feed_data('i\027[200~'..table.concat(t, '\n')..'\027[201~')
     screen:expect([[
       item 2997                                         |
       item 2998                                         |
@@ -180,7 +181,7 @@ describe('tui with non-tty file descriptors', function()
   it('can handle pipes as stdout and stderr', function()
     local screen = thelpers.screen_setup(0, '"'..helpers.nvim_prog
       ..' -u NONE -i NONE --cmd \'set noswapfile noshowcmd noruler\' --cmd \'normal iabc\' > /dev/null 2>&1 && cat testF && rm testF"')
-    feed(':w testF\n:q\n')
+    feed_data(':w testF\n:q\n')
     screen:expect([[
       :w testF                                          |
       :q                                                |
@@ -200,12 +201,13 @@ describe('tui focus event handling', function()
     helpers.clear()
     screen = thelpers.screen_setup(0, '["'..helpers.nvim_prog
       ..'", "-u", "NONE", "-i", "NONE", "--cmd", "set noswapfile noshowcmd noruler"]')
-    feed_command('autocmd FocusGained * echo "gained"')
+    feed_data(":autocmd FocusGained * echo 'gained'\n")
-    feed_command('autocmd FocusLost * echo "lost"')
+    feed_data(":autocmd FocusLost * echo 'lost'\n")
+    feed_data("\034\016")  -- CTRL-\ CTRL-N
   end)
 
   it('can handle focus events in normal mode', function()
-    feed('\027[I')
+    feed_data('\027[I')
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -216,7 +218,7 @@ describe('tui focus event handling', function()
       {3:-- TERMINAL --}                                    |
     ]])
 
-    feed('\027[O')
+    feed_data('\027[O')
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -230,8 +232,8 @@ describe('tui focus event handling', function()
 
   it('can handle focus events in insert mode', function()
     feed_command('set noshowmode')
-    feed('i')
+    feed_data('i')
-    feed('\027[I')
+    feed_data('\027[I')
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -241,7 +243,7 @@ describe('tui focus event handling', function()
       gained                                            |
       {3:-- TERMINAL --}                                    |
     ]])
-    feed('\027[O')
+    feed_data('\027[O')
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -254,8 +256,8 @@ describe('tui focus event handling', function()
   end)
 
   it('can handle focus events in cmdline mode', function()
-    feed(':')
+    feed_data(':')
-    feed('\027[I')
+    feed_data('\027[I')
     screen:expect([[
                                                         |
       {4:~                                                 }|
@@ -265,7 +267,7 @@ describe('tui focus event handling', function()
       g{1:a}ined                                            |
       {3:-- TERMINAL --}                                    |
     ]])
-    feed('\027[O')
+    feed_data('\027[O')
     screen:expect([[
                                                         |
       {4:~                                                 }|
@@ -278,30 +280,36 @@ describe('tui focus event handling', function()
   end)
 
   it('can handle focus events in terminal mode', function()
-    feed_command('set shell='..nvim_dir..'/shell-test')
+    feed_data(':set shell='..nvim_dir..'/shell-test\n')
-    feed_command('set laststatus=0')
+    feed_data(':set noshowmode laststatus=0\n')
-    feed_command('set noshowmode')
+
-    feed_command('terminal')
+    retry(2, 3 * screen.timeout, function()
-    feed('\027[I')
+      feed_data(':terminal\n')
-    screen:expect([[
+      feed_data('\027[I')
-      ready $                                           |
+      screen:expect([[
-      [Process exited 0]{1: }                               |
+        ready $                                           |
-                                                        |
+        [Process exited 0]{1: }                               |
-                                                        |
+                                                          |
-                                                        |
+                                                          |
-      gained                                            |
+                                                          |
-      {3:-- TERMINAL --}                                    |
+        gained                                            |
-    ]])
+        {3:-- TERMINAL --}                                    |
-   feed('\027[O')
+      ]])
-    screen:expect([[
+      feed_data('\027[O')
-      ready $                                           |
+      screen:expect([[
-      [Process exited 0]{1: }                               |
+        ready $                                           |
-                                                        |
+        [Process exited 0]{1: }                               |
-                                                        |
+                                                          |
-                                                        |
+                                                          |
-      lost                                              |
+                                                          |
-      {3:-- TERMINAL --}                                    |
+        lost                                              |
-    ]])
+        {3:-- TERMINAL --}                                    |
+      ]])
+
+      -- If retry is needed...
+      feed_data("\034\016")  -- CTRL-\ CTRL-N
+      feed_data(':bwipeout!\n')
+    end)
   end)
 end)
 

test/helpers.lua

@@ -17,18 +17,28 @@ local ok = function(res)
   return assert.is_true(res)
 end
 
+-- initial_path:  directory to recurse into
+-- re:            include pattern (string)
+-- exc_re:        exclude pattern(s) (string or table)
 local function glob(initial_path, re, exc_re)
+  exc_re = type(exc_re) == 'table' and exc_re or { exc_re }
   local paths_to_check = {initial_path}
   local ret = {}
   local checked_files = {}
+  local function is_excluded(path)
+    for _, pat in pairs(exc_re) do
+      if path:match(pat) then return true end
+    end
+    return false
+  end
+
   while #paths_to_check > 0 do
     local cur_path = paths_to_check[#paths_to_check]
     paths_to_check[#paths_to_check] = nil
     for e in lfs.dir(cur_path) do
       local full_path = cur_path .. '/' .. e
       local checked_path = full_path:sub(#initial_path + 1)
-      if ((not exc_re or not checked_path:match(exc_re))
+      if (not is_excluded(checked_path)) and e:sub(1, 1) ~= '.' then
-          and e:sub(1, 1) ~= '.') then
         local attrs = lfs.attributes(full_path)
         if attrs then
           local check_key = attrs.dev .. ':' .. tostring(attrs.ino)
@@ -106,13 +116,20 @@ local uname = (function()
   end)
 end)()
 
+local function tmpdir_get()
+  return os.getenv('TMPDIR') and os.getenv('TMPDIR') or os.getenv('TEMP')
+end
+
+-- Is temp directory `dir` defined local to the project workspace?
+local function tmpdir_is_local(dir)
+  return not not (dir and string.find(dir, 'Xtest'))
+end
+
 local tmpname = (function()
   local seq = 0
-  local tmpdir = os.getenv('TMPDIR') and os.getenv('TMPDIR') or os.getenv('TEMP')
+  local tmpdir = tmpdir_get()
-  -- Is $TMPDIR defined local to the project workspace?
-  local in_workspace = not not (tmpdir and string.find(tmpdir, 'Xtest'))
   return (function()
-    if in_workspace then
+    if tmpdir_is_local(tmpdir) then
       -- Cannot control os.tmpname() dir, so hack our own tmpname() impl.
       seq = seq + 1
       local fname = tmpdir..'/nvim-test-lua-'..seq
@@ -162,33 +179,34 @@ end
 
 local tests_skipped = 0
 
-local function check_cores(app)
+local function check_cores(app, force)
   app = app or 'build/bin/nvim'
   local initial_path, re, exc_re
   local gdb_db_cmd = 'gdb -n -batch -ex "thread apply all bt full" "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"'
   local lldb_db_cmd = 'lldb -Q -o "bt all" -f "$_NVIM_TEST_APP" -c "$_NVIM_TEST_CORE"'
   local random_skip = false
+  local local_tmpdir = tmpdir_is_local(tmpdir_get()) and tmpdir_get() or nil
   local db_cmd
   if hasenv('NVIM_TEST_CORE_GLOB_DIRECTORY') then
     initial_path = os.getenv('NVIM_TEST_CORE_GLOB_DIRECTORY')
     re = os.getenv('NVIM_TEST_CORE_GLOB_RE')
-    exc_re = os.getenv('NVIM_TEST_CORE_EXC_RE')
+    exc_re = { os.getenv('NVIM_TEST_CORE_EXC_RE'), local_tmpdir }
     db_cmd = os.getenv('NVIM_TEST_CORE_DB_CMD') or gdb_db_cmd
     random_skip = os.getenv('NVIM_TEST_CORE_RANDOM_SKIP')
   elseif os.getenv('TRAVIS_OS_NAME') == 'osx' then
     initial_path = '/cores'
     re = nil
-    exc_re = nil
+    exc_re = { local_tmpdir }
     db_cmd = lldb_db_cmd
   else
     initial_path = '.'
     re = '/core[^/]*$'
-    exc_re = '^/%.deps$'
+    exc_re = { '^/%.deps$', local_tmpdir }
     db_cmd = gdb_db_cmd
     random_skip = true
   end
   -- Finding cores takes too much time on linux
-  if random_skip and math.random() < 0.9 then
+  if not force and random_skip and math.random() < 0.9 then
     tests_skipped = tests_skipped + 1
     return
   end

test/unit/eval/decode_spec.lua

@@ -2,7 +2,6 @@ local helpers = require('test.unit.helpers')(after_each)
 local itp = helpers.gen_itp(it)
 
 local cimport = helpers.cimport
-local to_cstr = helpers.to_cstr
 local eq = helpers.eq
 local neq = helpers.neq
 local ffi = helpers.ffi
@@ -72,7 +71,7 @@ describe('json_decode_string()', function()
   end
 
   itp('does not overflow in error messages', function()
-    local saved_p_enc = decode.p_enc
+    collectgarbage('restart')
     check_failure(']test', 1, 'E474: No container to close: ]')
     check_failure('[}test', 2, 'E474: Closing list with curly bracket: }')
     check_failure('{]test', 2,
@@ -105,10 +104,6 @@ describe('json_decode_string()', function()
     check_failure('"\194"test', 3, 'E474: Only UTF-8 strings allowed: \194"')
     check_failure('"\252\144\128\128\128\128"test', 8, 'E474: Only UTF-8 code points up to U+10FFFF are allowed to appear unescaped: \252\144\128\128\128\128"')
     check_failure('"test', 1, 'E474: Expected string end: "')
-    decode.p_enc = to_cstr('latin1')
-    check_failure('"\\uABCD"test', 8,
-                  'E474: Failed to convert string "ꯍ" from UTF-8')
-    decode.p_enc = saved_p_enc
     check_failure('-test', 1, 'E474: Missing number after minus sign: -')
     check_failure('-1.test', 3, 'E474: Missing number after decimal dot: -1.')
     check_failure('-1.0etest', 5, 'E474: Missing exponent: -1.0e')

test/unit/eval/helpers.lua

@@ -468,7 +468,7 @@ local function tbl2callback(tbl)
                                    data={funcref=eval.xstrdup(tbl.fref)}}})
   elseif tbl.type == 'pt' then
     local pt = ffi.gc(ffi.cast('partial_T*',
-        eval.xcalloc(1, ffi.sizeof('partial_T'))), eval.partial_unref)
+        eval.xcalloc(1, ffi.sizeof('partial_T'))), nil)
     ret = ffi.new('Callback[1]', {{type=eval.kCallbackPartial,
                                    data={partial=populate_partial(pt, tbl.pt, {})}}})
   else

test/unit/eval/typval_spec.lua

@@ -47,6 +47,15 @@ local lib = cimport('./src/nvim/eval/typval.h', './src/nvim/memory.h',
                     './src/nvim/eval.h', './src/nvim/vim.h',
                     './src/nvim/globals.h')
 
+local function vimconv_alloc()
+  return ffi.gc(
+    ffi.cast('vimconv_T*', lib.xcalloc(1, ffi.sizeof('vimconv_T'))),
+    function(vc)
+      lib.convert_setup(vc, nil, nil)
+      lib.xfree(vc)
+    end)
+end
+
 local function list_watch_alloc(li)
   return ffi.cast('listwatch_T*', ffi.new('listwatch_T[1]', {{lw_item=li}}))
 end
@@ -237,24 +246,33 @@ describe('typval.c', function()
             list_watch(l, lis[4]),
             list_watch(l, lis[7]),
           }
+          alloc_log:check({
+            a.list(l),
+            a.li(lis[1]),
+            a.li(lis[2]),
+            a.li(lis[3]),
+            a.li(lis[4]),
+            a.li(lis[5]),
+            a.li(lis[6]),
+            a.li(lis[7]),
+          })
 
           lib.tv_list_item_remove(l, lis[4])
-          ffi.gc(lis[4], lib.tv_list_item_free)
+          alloc_log:check({a.freed(lis[4])})
           eq({lis[1], lis[5], lis[7]}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item})
 
           lib.tv_list_item_remove(l, lis[2])
-          ffi.gc(lis[2], lib.tv_list_item_free)
+          alloc_log:check({a.freed(lis[2])})
           eq({lis[1], lis[5], lis[7]}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item})
 
           lib.tv_list_item_remove(l, lis[7])
-          ffi.gc(lis[7], lib.tv_list_item_free)
+          alloc_log:check({a.freed(lis[7])})
           eq({lis[1], lis[5], nil}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item == nil and nil})
 
           lib.tv_list_item_remove(l, lis[1])
-          ffi.gc(lis[1], lib.tv_list_item_free)
+          alloc_log:check({a.freed(lis[1])})
           eq({lis[3], lis[5], nil}, {lws[1].lw_item, lws[2].lw_item, lws[3].lw_item == nil and nil})
 
-          alloc_log:clear()
           lib.tv_list_watch_remove(l, lws[2])
           lib.tv_list_watch_remove(l, lws[3])
           lib.tv_list_watch_remove(l, lws[1])
@@ -460,6 +478,10 @@ describe('typval.c', function()
         eq(empty_list, typvalt2lua(l_tv))
         eq({true, true, true}, {lws[1].lw_item == nil, lws[2].lw_item == nil, lws[3].lw_item == nil})
 
+        lib.tv_list_watch_remove(l, lws[1])
+        lib.tv_list_watch_remove(l, lws[2])
+        lib.tv_list_watch_remove(l, lws[3])
+
         alloc_log:check({})
       end)
     end)
@@ -730,10 +752,8 @@ describe('typval.c', function()
         collectgarbage()
       end)
       itp('copies list correctly and converts items', function()
-        local vc = ffi.gc(ffi.new('vimconv_T[1]'), function(vc)
+        local vc = vimconv_alloc()
-          lib.convert_setup(vc, nil, nil)
+        -- UTF-8 ↔ latin1 conversions needs no iconv
-        end)
-        -- UTF-8 ↔ latin1 conversions need no iconv
         eq(OK, lib.convert_setup(vc, to_cstr('utf-8'), to_cstr('latin1')))
 
         local v = {{['«']='»'}, {'„'}, 1, '“', null_string, null_list, null_dict}
@@ -1087,12 +1107,16 @@ describe('typval.c', function()
       end)
     end)
     describe('join()', function()
-      local function list_join(l, sep, ret)
+      local function list_join(l, sep, join_ret)
         local ga = ga_alloc()
-        eq(ret or OK, lib.tv_list_join(ga, l, sep))
+        eq(join_ret or OK, lib.tv_list_join(ga, l, sep))
-        if ga.ga_data == nil then return ''
+        local ret = ''
-        else return ffi.string(ga.ga_data)
+        if ga.ga_data ~= nil then
+          ret = ffi.string(ga.ga_data)
         end
+        -- For some reason this is not working well in GC
+        lib.ga_clear(ffi.gc(ga, nil))
+        return ret
       end
       itp('works', function()
         local l
@@ -1508,7 +1532,7 @@ describe('typval.c', function()
           eq(s:sub(1, len), ffi.string(di.di_key))
           alloc_log:check({a.di(di, len)})
           if tv then
-            di.di_tv = tv
+            di.di_tv = ffi.gc(tv, nil)
           else
             di.di_tv.v_type = lib.VAR_UNKNOWN
           end
@@ -1539,7 +1563,7 @@ describe('typval.c', function()
           alloc_log:check({a.dict(d)})
           local di = ffi.gc(lib.tv_dict_item_alloc(''), nil)
           local tv = lua2typvalt('test')
-          di.di_tv = tv
+          di.di_tv = ffi.gc(tv, nil)
           alloc_log:check({a.di(di, ''), a.str(tv.vval.v_string, 'test')})
           eq(OK, lib.tv_dict_add(d, di))
           alloc_log:check({})
@@ -2131,9 +2155,7 @@ describe('typval.c', function()
         collectgarbage()
       end)
       itp('copies dict correctly and converts items', function()
-        local vc = ffi.gc(ffi.new('vimconv_T[1]'), function(vc)
+        local vc = vimconv_alloc()
-          lib.convert_setup(vc, nil, nil)
-        end)
         -- UTF-8 ↔ latin1 conversions need no iconv
         eq(OK, lib.convert_setup(vc, to_cstr('utf-8'), to_cstr('latin1')))
 
@@ -2659,7 +2681,8 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 0},
             {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_number(UNKNOWN)', 0},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]
@@ -2687,7 +2710,8 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 0},
             {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_number(UNKNOWN)', 0},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = {v[4], not not emsg}
@@ -2721,7 +2745,8 @@ describe('typval.c', function()
             {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_number(UNKNOWN)', -1},
           }) do
             lib.curwin.w_cursor.lnum = 46
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]
@@ -2749,7 +2774,8 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, 'E907: Using a special value as a Float', 0},
             {lib.VAR_UNKNOWN, nil, 'E685: Internal error: tv_get_float(UNKNOWN)', 0},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]
@@ -2780,7 +2806,9 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'},
             {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', ''},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr in place of Neovim allocated string, cannot
+            -- tv_clear() that.
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]
@@ -2821,7 +2849,8 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'},
             {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', nil},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]
@@ -2861,7 +2890,8 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'},
             {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', ''},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]
@@ -2902,7 +2932,8 @@ describe('typval.c', function()
             {lib.VAR_SPECIAL, {v_special=lib.kSpecialVarFalse}, nil, 'false'},
             {lib.VAR_UNKNOWN, nil, 'E908: using an invalid value as a String', nil},
           }) do
-            local tv = typvalt(v[1], v[2])
+            -- Using to_cstr, cannot free with tv_clear
+            local tv = ffi.gc(typvalt(v[1], v[2]), nil)
             alloc_log:check({})
             local emsg = v[3]
             local ret = v[4]

test/unit/helpers.lua

@@ -632,8 +632,9 @@ local function itp_child(wr, func)
   collectgarbage('stop')
   child_sethook(wr)
   local err, emsg = pcall(func)
-  debug.sethook()
   collectgarbage('restart')
+  collectgarbage()
+  debug.sethook()
   emsg = tostring(emsg)
   sc.write(wr, trace_end_msg)
   if not err then
@@ -654,6 +655,7 @@ end
 
 local function check_child_err(rd)
   local trace = {}
+  local did_traceline = false
   while true do
     local traceline = sc.read(rd, hook_msglen)
     if #traceline ~= hook_msglen then
@@ -664,6 +666,7 @@ local function check_child_err(rd)
       end
     end
     if traceline == trace_end_msg then
+      did_traceline = true
       break
     end
     trace[#trace + 1] = traceline
@@ -679,6 +682,13 @@ local function check_child_err(rd)
         error = error .. trace[i]
       end
     end
+    if not did_traceline then
+      error = error .. '\nNo end of trace occurred'
+    end
+    local cc_err, cc_emsg = pcall(check_cores, Paths.test_luajit_prg, true)
+    if not cc_err then
+      error = error .. '\ncheck_cores failed: ' .. cc_emsg
+    end
     assert.just_fail(error)
   end
   if res == '+\n' then
@@ -764,11 +774,6 @@ local module = {
   child_cleanup_once = child_cleanup_once,
   sc = sc,
 }
-return function(after_each)
+return function()
-  if after_each then
-    after_each(function()
-      check_cores(Paths.test_luajit_prg)
-    end)
-  end
   return module
 end