os/env: use libuv v1.12 getenv/setenv API

- Minimum required libuv is now v1.12
- Because `uv_os_getenv` requires allocating, we must manage a map
  (`envmap` in `env.c`) to maintain the old behavior of `os_getenv` .
- free() map-items after removal. khash.h does not make copies of
  anything, so even its keys must be memory-managed by the caller.

closes #8398
closes #9267

CMakeLists.txt

@@ -379,7 +379,7 @@ endif()
 include_directories("${PROJECT_BINARY_DIR}/config")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
-find_package(LibUV REQUIRED)
+find_package(LibUV REQUIRED)  # minimum version: v1.12
 include_directories(SYSTEM ${LIBUV_INCLUDE_DIRS})
 
 find_package(Msgpack 1.0.0 REQUIRED)

config/CMakeLists.txt

@@ -47,17 +47,8 @@ if(Iconv_FOUND)
   set(HAVE_ICONV 1)
 endif()
 
-check_function_exists(_putenv_s HAVE_PUTENV_S)
-if(WIN32 AND NOT HAVE_PUTENV_S)
-  message(SEND_ERROR "_putenv_s() function not found on your system.")
-endif()
 check_function_exists(opendir HAVE_OPENDIR)
 check_function_exists(readlink HAVE_READLINK)
-check_function_exists(setenv HAVE_SETENV)
-if(UNIX AND NOT HAVE_SETENV)
-  message(SEND_ERROR "setenv() function not found on your system.")
-endif()
-check_function_exists(unsetenv HAVE_UNSETENV)
 check_function_exists(setpgid HAVE_SETPGID)
 check_function_exists(setsid HAVE_SETSID)
 check_function_exists(sigaction HAVE_SIGACTION)

config/config.h.in

@@ -32,8 +32,6 @@
 #cmakedefine HAVE_UV_TRANSLATE_SYS_ERROR
 // TODO: add proper cmake check
 // #define HAVE_SELINUX 1
-#cmakedefine HAVE_SETENV
-#cmakedefine HAVE_UNSETENV
 #cmakedefine HAVE_SETPGID
 #cmakedefine HAVE_SETSID
 #cmakedefine HAVE_SIGACTION

src/nvim/hashtab.h

@@ -19,7 +19,7 @@ typedef size_t hash_T;
 #define HASHITEM_EMPTY(hi) ((hi)->hi_key == NULL \
                             || (hi)->hi_key == (char_u *)&hash_removed)
 
-/// A hastable item.
+/// Hashtable item.
 ///
 /// Each item has a NUL terminated string key.
 /// A key can appear only once in the table.

src/nvim/lib/khash.h

@@ -24,24 +24,24 @@
 */
 
 /*
-  An example:
+  Example:
 
 #include "nvim/khash.h"
 KHASH_MAP_INIT_INT(32, char)
 int main() {
-	int ret, is_missing;
+    int ret, is_missing;
-	khiter_t k;
+    khiter_t k;
-	khash_t(32) *h = kh_init(32);
+    khash_t(32) *h = kh_init(32);
-	k = kh_put(32, h, 5, &ret);
+    k = kh_put(32, h, 5, &ret);
-	kh_value(h, k) = 10;
+    kh_value(h, k) = 10;
-	k = kh_get(32, h, 10);
+    k = kh_get(32, h, 10);
-	is_missing = (k == kh_end(h));
+    is_missing = (k == kh_end(h));
-	k = kh_get(32, h, 5);
+    k = kh_get(32, h, 5);
-	kh_del(32, h, k);
+    kh_del(32, h, k);
-	for (k = kh_begin(h); k != kh_end(h); ++k)
+    for (k = kh_begin(h); k != kh_end(h); ++k)
-		if (kh_exist(h, k)) kh_value(h, k) = 1;
+        if (kh_exist(h, k)) kh_value(h, k) = 1;
-	kh_destroy(32, h);
+    kh_destroy(32, h);
-	return 0;
+    return 0;
 }
 */
 
@@ -539,7 +539,7 @@ static kh_inline khint_t __ac_Wang_hash(khint_t key)
   @param  r     Extra return code: -1 if the operation failed;
                 0 if the key is present in the hash table;
                 1 if the bucket is empty (never used); 2 if the element in
-				the bucket has been deleted [int*]
+                the bucket has been deleted [int*]
   @return       Iterator to the inserted element [khint_t]
  */
 #define kh_put(name, h, k, r) kh_put_##name(h, k, r)

src/nvim/main.c

@@ -184,6 +184,7 @@ bool event_teardown(void)
 void early_init(void)
 {
   log_init();
+  env_init();
   fs_init();
   handle_init();
   eval_init();          // init global variables
@@ -1769,7 +1770,7 @@ static bool do_user_initialization(void)
   FUNC_ATTR_WARN_UNUSED_RESULT
 {
   bool do_exrc = p_exrc;
-  if (process_env("VIMINIT") == OK) {
+  if (execute_env("VIMINIT") == OK) {
     do_exrc = p_exrc;
     return do_exrc;
   }
@@ -1814,7 +1815,7 @@ static bool do_user_initialization(void)
     } while (iter != NULL);
     xfree(config_dirs);
   }
-  if (process_env("EXINIT") == OK) {
+  if (execute_env("EXINIT") == OK) {
     do_exrc = p_exrc;
     return do_exrc;
   }
@@ -1878,7 +1879,7 @@ static void source_startup_scripts(const mparm_T *const parmp)
 ///
 /// @return FAIL if the environment variable was not executed,
 ///         OK otherwise.
-static int process_env(char *env)
+static int execute_env(char *env)
   FUNC_ATTR_NONNULL_ALL
 {
   const char *initstr = os_getenv(env);

src/nvim/map.c

@@ -1,6 +1,13 @@
 // This is an open source non-commercial project. Dear PVS-Studio, please check
 // it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
 
+///
+/// map.c: khash.h wrapper
+///
+/// NOTE: Callers must manage memory (allocate) for keys and values.
+///       khash.h does not make its own copy of the key or value.
+///
+
 #include <stdlib.h>
 #include <stdbool.h>
 #include <string.h>
@@ -72,6 +79,16 @@
     return kh_get(T##_##U##_map, map->table, key) != kh_end(map->table); \
   } \
   \
+  T map_##T##_##U##_key(Map(T, U) *map, T key) \
+  { \
+    khiter_t k; \
+    \
+    if ((k = kh_get(T##_##U##_map, map->table, key)) == kh_end(map->table)) { \
+      abort();  /* Caller must check map_has(). */ \
+    } \
+    \
+    return kh_key(map->table, k); \
+  } \
   U map_##T##_##U##_put(Map(T, U) *map, T key, U value) \
   { \
     int ret; \
@@ -167,3 +184,18 @@ MAP_IMPL(String, MsgpackRpcRequestHandler, MSGPACK_HANDLER_INITIALIZER)
 #define KVEC_INITIALIZER { .size = 0, .capacity = 0, .items = NULL }
 MAP_IMPL(HlEntry, int, DEFAULT_INITIALIZER)
 MAP_IMPL(String, handle_T, 0)
+
+
+/// Deletes a key:value pair from a string:pointer map, and frees the
+/// storage of both key and value.
+///
+void pmap_del2(PMap(cstr_t) *map, const char *key)
+{
+  if (pmap_has(cstr_t)(map, key)) {
+    void *k = (void *)pmap_key(cstr_t)(map, key);
+    void *v = pmap_get(cstr_t)(map, key);
+    pmap_del(cstr_t)(map, key);
+    xfree(k);
+    xfree(v);
+  }
+}

src/nvim/map.h

@@ -25,11 +25,15 @@
   void map_##T##_##U##_free(Map(T, U) *map); \
   U map_##T##_##U##_get(Map(T, U) *map, T key); \
   bool map_##T##_##U##_has(Map(T, U) *map, T key); \
+  T map_##T##_##U##_key(Map(T, U) *map, T key); \
   U map_##T##_##U##_put(Map(T, U) *map, T key, U value); \
   U *map_##T##_##U##_ref(Map(T, U) *map, T key, bool put); \
   U map_##T##_##U##_del(Map(T, U) *map, T key); \
   void map_##T##_##U##_clear(Map(T, U) *map);
 
+//
+// NOTE: Keys AND values must be allocated! khash.h does not make a copy.
+//
 MAP_DECLS(int, int)
 MAP_DECLS(cstr_t, ptr_t)
 MAP_DECLS(ptr_t, ptr_t)
@@ -43,6 +47,7 @@ MAP_DECLS(String, handle_T)
 #define map_free(T, U) map_##T##_##U##_free
 #define map_get(T, U) map_##T##_##U##_get
 #define map_has(T, U) map_##T##_##U##_has
+#define map_key(T, U) map_##T##_##U##_key
 #define map_put(T, U) map_##T##_##U##_put
 #define map_ref(T, U) map_##T##_##U##_ref
 #define map_del(T, U) map_##T##_##U##_del
@@ -52,7 +57,9 @@ MAP_DECLS(String, handle_T)
 #define pmap_free(T) map_free(T, ptr_t)
 #define pmap_get(T) map_get(T, ptr_t)
 #define pmap_has(T) map_has(T, ptr_t)
+#define pmap_key(T) map_key(T, ptr_t)
 #define pmap_put(T) map_put(T, ptr_t)
+/// @see pmap_del2
 #define pmap_del(T) map_del(T, ptr_t)
 #define pmap_clear(T) map_clear(T, ptr_t)
 
@@ -62,4 +69,6 @@ MAP_DECLS(String, handle_T)
 #define map_foreach_value(map, value, block) \
   kh_foreach_value(map->table, value, block)
 
+void pmap_del2(PMap(cstr_t) *map, const char *key);
+
 #endif  // NVIM_MAP_H

src/nvim/os/env.c

@@ -19,6 +19,7 @@
 #include "nvim/eval.h"
 #include "nvim/ex_getln.h"
 #include "nvim/version.h"
+#include "nvim/map.h"
 
 #ifdef WIN32
 #include "nvim/mbyte.h"  // for utf8_to_utf16, utf16_to_utf8
@@ -32,53 +33,119 @@
 #include <sys/utsname.h>
 #endif
 
+// Because `uv_os_getenv` requires allocating, we must manage a map to maintain
+// the behavior of `os_getenv`.
+static PMap(cstr_t) *envmap;
+static uv_mutex_t mutex;
+
+void env_init(void)
+{
+  envmap = pmap_new(cstr_t)();
+  uv_mutex_init(&mutex);
+}
+
 /// Like getenv(), but returns NULL if the variable is empty.
 const char *os_getenv(const char *name)
   FUNC_ATTR_NONNULL_ALL
 {
-  const char *e = getenv(name);
+  char *e;
-  return e == NULL || *e == NUL ? NULL : e;
+  size_t size = 64;
+  if (name[0] == '\0') {
+    return NULL;
+  }
+  uv_mutex_lock(&mutex);
+  if (pmap_has(cstr_t)(envmap, name)
+      && !!(e = (char *)pmap_get(cstr_t)(envmap, name))) {
+    if (e[0] != '\0') {
+      // Found non-empty cached env var.
+      // NOTE: This risks incoherence if an in-process library changes the
+      //       environment without going through our os_setenv() wrapper.  If
+      //       that turns out to be a problem, we can just remove this codepath.
+      goto end;
+    }
+    pmap_del2(envmap, name);
+  }
+  e = xmalloc(size);
+  int r = uv_os_getenv(name, e, &size);
+  if (r == UV_ENOBUFS) {
+    e = xrealloc(e, size);
+    r = uv_os_getenv(name, e, &size);
+  }
+  if (r != 0 || size == 0 || e[0] == '\0') {
+    xfree(e);
+    e = NULL;
+    if (r != 0 && r != UV_ENOENT && r != UV_UNKNOWN) {
+      ELOG("uv_os_getenv(%s) failed: %d %s", name, r, uv_err_name(r));
+    }
+    goto end;
+  }
+  pmap_put(cstr_t)(envmap, xstrdup(name), e);
+end:
+  uv_mutex_unlock(&mutex);
+  return (e == NULL || size == 0 || e[0] == '\0') ? NULL : e;
 }
 
-/// Returns `true` if the environment variable, `name`, has been defined
+/// Returns true if environment variable `name` is defined (even if empty).
-/// (even if empty).
+/// Returns false if not found (UV_ENOENT) or other failure.
 bool os_env_exists(const char *name)
   FUNC_ATTR_NONNULL_ALL
 {
-  return getenv(name) != NULL;
+  if (name[0] == '\0') {
+    return false;
+  }
+  // Use a tiny buffer because we don't care about the value: if uv_os_getenv()
+  // returns UV_ENOBUFS, the env var was found.
+  char buf[1];
+  size_t size = sizeof(buf);
+  int r = uv_os_getenv(name, buf, &size);
+  assert(r != UV_EINVAL);
+  if (r != 0 && r != UV_ENOENT && r != UV_ENOBUFS) {
+    ELOG("uv_os_getenv(%s) failed: %d %s", name, r, uv_err_name(r));
+  }
+  return (r == 0 || r == UV_ENOBUFS);
 }
 
 int os_setenv(const char *name, const char *value, int overwrite)
   FUNC_ATTR_NONNULL_ALL
 {
+  if (name[0] == '\0') {
+    return -1;
+  }
 #ifdef WIN32
   if (!overwrite && os_getenv(name) != NULL) {
     return 0;
   }
-  int rv = (int)_putenv_s(name, value);
-  if (rv != 0) {
-    ELOG("_putenv_s failed: %d: %s", rv, uv_strerror(rv));
-    return -1;
-  }
-  return 0;
-#elif defined(HAVE_SETENV)
-  return setenv(name, value, overwrite);
 #else
-# error "This system has no implementation available for os_setenv()"
+  if (!overwrite && os_env_exists(name)) {
+    return 0;
+  }
 #endif
+  uv_mutex_lock(&mutex);
+  pmap_del2(envmap, name);
+  int r = uv_os_setenv(name, value);
+  assert(r != UV_EINVAL);
+  if (r != 0) {
+    ELOG("uv_os_setenv(%s) failed: %d %s", name, r, uv_err_name(r));
+  }
+  uv_mutex_unlock(&mutex);
+  return r == 0 ? 0 : -1;
 }
 
 /// Unset environment variable
-///
-/// For systems where unsetenv() is not available the value will be set as an
-/// empty string
 int os_unsetenv(const char *name)
+  FUNC_ATTR_NONNULL_ALL
 {
-#ifdef HAVE_UNSETENV
+  if (name[0] == '\0') {
-  return unsetenv(name);
+    return -1;
-#else
+  }
-  return os_setenv(name, "", 1);
+  uv_mutex_lock(&mutex);
-#endif
+  pmap_del2(envmap, name);
+  int r = uv_os_unsetenv(name);
+  if (r != 0) {
+    ELOG("uv_os_unsetenv(%s) failed: %d %s", name, r, uv_err_name(r));
+  }
+  uv_mutex_unlock(&mutex);
+  return r == 0 ? 0 : -1;
 }
 
 char *os_getenvname_at_index(size_t index)
@@ -515,7 +582,7 @@ static char *remove_tail(char *path, char *pend, char *dirname)
   return pend;
 }
 
-/// Iterate over a delimited list.
+/// Iterates $PATH-like delimited list `val`.
 ///
 /// @note Environment variables must not be modified during iteration.
 ///
@@ -550,7 +617,7 @@ const void *vim_env_iter(const char delim,
   }
 }
 
-/// Iterate over a delimited list in reverse order.
+/// Iterates $PATH-like delimited list `val` in reverse order.
 ///
 /// @note Environment variables must not be modified during iteration.
 ///
@@ -587,11 +654,12 @@ const void *vim_env_iter_rev(const char delim,
   }
 }
 
-/// Vim's version of getenv().
+/// Vim getenv() wrapper with special handling of $HOME, $VIM, $VIMRUNTIME,
-/// Special handling of $HOME, $VIM and $VIMRUNTIME, allowing the user to
+/// allowing the user to override the Nvim runtime directory at runtime.
-/// override the vim runtime directory at runtime.  Also does ACP to 'enc'
+/// Result must be freed by the caller.
-/// conversion for Win32.  Result must be freed by the caller.
+///
 /// @param name Environment variable to expand
+/// @return [allocated] Expanded environment variable, or NULL
 char *vim_getenv(const char *name)
 {
   // init_path() should have been called before now.
@@ -857,9 +925,8 @@ char_u * home_replace_save(buf_T *buf, char_u *src) FUNC_ATTR_NONNULL_RET
   return dst;
 }
 
-/// Our portable version of setenv.
+/// Vim setenv() wrapper with special handling for $VIMRUNTIME to keep the
-/// Has special handling for $VIMRUNTIME to keep the localization machinery
+/// localization machinery sane.
-/// sane.
 void vim_setenv(const char *name, const char *val)
 {
   os_setenv(name, val, 1);

src/nvim/os/fs_defs.h

@@ -21,9 +21,7 @@ typedef struct {
   uv_dirent_t ent;  ///< @private The entry information.
 } Directory;
 
-/// Function to convert libuv error to char * error description
+/// Converts libuv error (negative int) to error description string.
-///
-/// negative libuv error codes are returned by a number of os functions.
 #define os_strerror uv_strerror
 
 // Values returned by os_nodetype()

src/nvim/os/pty_process_unix.c

@@ -177,7 +177,7 @@ static void init_child(PtyProcess *ptyproc)
   }
 
   char *prog = ptyproc->process.argv[0];
-  setenv("TERM", ptyproc->term_name ? ptyproc->term_name : "ansi", 1);
+  os_setenv("TERM", ptyproc->term_name ? ptyproc->term_name : "ansi", 1);
   execvp(prog, ptyproc->process.argv);
   ELOG("execvp failed: %s: %s", strerror(errno), prog);
   _exit(122);  // 122 is EXEC_FAILED in the Vim source.

src/nvim/path.c

@@ -2271,7 +2271,7 @@ int path_is_absolute(const char_u *fname)
 void path_guess_exepath(const char *argv0, char *buf, size_t bufsize)
   FUNC_ATTR_NONNULL_ALL
 {
-  char *path = getenv("PATH");
+  const char *path = os_getenv("PATH");
 
   if (path == NULL || path_is_absolute((char_u *)argv0)) {
     xstrlcpy(buf, argv0, bufsize);

test/unit/helpers.lua

@@ -645,16 +645,16 @@ local function itp_child(wr, func)
     s = s:sub(1, hook_msglen - 2)
     sc.write(wr, '>' .. s .. (' '):rep(hook_msglen - 2 - #s) .. '\n')
   end
-  local err, emsg = pcall(init)
+  local status, result = pcall(init)
-  if err then
+  if status then
     collectgarbage('stop')
     child_sethook(wr)
-    err, emsg = pcall(func)
+    status, result = pcall(func)
     debug.sethook()
   end
-  emsg = tostring(emsg)
   sc.write(wr, trace_end_msg)
-  if not err then
+  if not status then
+    local emsg = tostring(result)
     if #emsg > 99999 then
       emsg = emsg:sub(1, 99999)
     end
@@ -668,7 +668,7 @@ local function itp_child(wr, func)
   collectgarbage()
   sc.write(wr, '$\n')
   sc.close(wr)
-  sc.exit(err and 0 or 1)
+  sc.exit(status and 0 or 1)
 end
 
 local function check_child_err(rd)

test/unit/os/env_spec.lua

@@ -8,17 +8,22 @@ local ffi = helpers.ffi
 local cstr = helpers.cstr
 local to_cstr = helpers.to_cstr
 local NULL = helpers.NULL
+local OK = 0
 
 require('lfs')
 
 local cimp = cimport('./src/nvim/os/os.h')
 
 describe('env.c', function()
+  local function os_env_exists(name)
+    return cimp.os_env_exists(to_cstr(name))
+  end
+
   local function os_setenv(name, value, override)
     return cimp.os_setenv(to_cstr(name), to_cstr(value), override)
   end
 
-  local function os_unsetenv(name, _, _)
+  local function os_unsetenv(name)
     return cimp.os_unsetenv(to_cstr(name))
   end
 
@@ -31,25 +36,44 @@ describe('env.c', function()
     end
   end
 
-  describe('os_setenv', function()
+  itp('os_env_exists', function()
-    local OK = 0
+    eq(false, os_env_exists(''))
+    eq(false, os_env_exists('      '))
+    eq(false, os_env_exists('\t'))
+    eq(false, os_env_exists('\n'))
+    eq(false, os_env_exists('AaあB <= very weird name...'))
 
-    itp('sets an env variable and returns OK', function()
+    local varname = 'NVIM_UNIT_TEST_os_env_exists'
+    eq(false, os_env_exists(varname))
+    eq(OK, os_setenv(varname, 'foo bar baz ...', 1))
+    eq(true, os_env_exists(varname))
+  end)
+
+  describe('os_setenv', function()
+    itp('sets an env var and returns success', function()
       local name = 'NVIM_UNIT_TEST_SETENV_1N'
       local value = 'NVIM_UNIT_TEST_SETENV_1V'
       eq(nil, os.getenv(name))
-      eq(OK, (os_setenv(name, value, 1)))
+      eq(OK, os_setenv(name, value, 1))
       eq(value, os.getenv(name))
+
+      -- Set empty, then set non-empty, then retrieve.
+      eq(OK, os_setenv(name, '', 1))
+      eq('', os.getenv(name))
+      eq(OK, os_setenv(name, 'non-empty', 1))
+      eq('non-empty', os.getenv(name))
     end)
 
-    itp("dosn't overwrite an env variable if overwrite is 0", function()
+    itp("`overwrite` behavior", function()
       local name = 'NVIM_UNIT_TEST_SETENV_2N'
       local value = 'NVIM_UNIT_TEST_SETENV_2V'
       local value_updated = 'NVIM_UNIT_TEST_SETENV_2V_UPDATED'
-      eq(OK, (os_setenv(name, value, 0)))
+      eq(OK, os_setenv(name, value, 0))
       eq(value, os.getenv(name))
-      eq(OK, (os_setenv(name, value_updated, 0)))
+      eq(OK, os_setenv(name, value_updated, 0))
       eq(value, os.getenv(name))
+      eq(OK, os_setenv(name, value_updated, 1))
+      eq(value_updated, os.getenv(name))
     end)
   end)
 
@@ -93,31 +117,42 @@ describe('env.c', function()
   end)
 
   describe('os_getenv', function()
-    itp('reads an env variable', function()
+    itp('reads an env var', function()
       local name = 'NVIM_UNIT_TEST_GETENV_1N'
       local value = 'NVIM_UNIT_TEST_GETENV_1V'
       eq(NULL, os_getenv(name))
       -- Use os_setenv because Lua dosen't have setenv.
       os_setenv(name, value, 1)
       eq(value, os_getenv(name))
+
+      -- Get a big value.
+      local bigval = ('x'):rep(256)
+      eq(OK, os_setenv(name, bigval, 1))
+      eq(bigval, os_getenv(name))
+
+      -- Set non-empty, then set empty.
+      eq(OK, os_setenv(name, 'non-empty', 1))
+      eq('non-empty', os_getenv(name))
+      eq(OK, os_setenv(name, '', 1))
+      eq(NULL, os_getenv(name))
     end)
 
-    itp('returns NULL if the env variable is not found', function()
+    itp('returns NULL if the env var is not found', function()
-      local name = 'NVIM_UNIT_TEST_GETENV_NOTFOUND'
+      eq(NULL, os_getenv('NVIM_UNIT_TEST_GETENV_NOTFOUND'))
-      return eq(NULL, os_getenv(name))
     end)
   end)
 
-  describe('os_unsetenv', function()
+  itp('os_unsetenv', function()
-    itp('unsets environment variable', function()
+    local name = 'TEST_UNSETENV'
-      local name = 'TEST_UNSETENV'
+    local value = 'TESTVALUE'
-      local value = 'TESTVALUE'
+    os_setenv(name, value, 1)
-      os_setenv(name, value, 1)
+    eq(OK, os_unsetenv(name))
-      os_unsetenv(name)
+    neq(os_getenv(name), value)
-      neq(os_getenv(name), value)
+    -- Depending on the platform the var might be unset or set as ''
-      -- Depending on the platform the var might be unset or set as ''
+    assert.True(os_getenv(name) == nil or os_getenv(name) == '')
-      assert.True(os_getenv(name) == nil or os_getenv(name) == '')
+    if os_getenv(name) == nil then
-    end)
+      eq(false, os_env_exists(name))
+    end
   end)
 
   describe('os_getenvname_at_index', function()