Change to use ConPTY, if available

src/nvim/CMakeLists.txt

@@ -134,6 +134,9 @@ foreach(sfile ${NVIM_SOURCES})
   if(NOT WIN32 AND ${f} MATCHES "^(pty_process_win.c)$")
     list(APPEND to_remove ${sfile})
   endif()
+  if(NOT WIN32 AND ${f} MATCHES "^(os_win_conpty.c)$")
+    list(APPEND to_remove ${sfile})
+  endif()
 endforeach()
 
 list(REMOVE_ITEM NVIM_SOURCES ${to_remove})
@@ -637,6 +640,7 @@ endfunction()
 set(NO_SINGLE_CHECK_HEADERS
   os/win_defs.h
   os/pty_process_win.h
+  os/os_win_conpty.h
 )
 foreach(hfile ${NVIM_HEADERS})
   get_test_target(test-includes "${hfile}" relative_path texe)

src/nvim/channel.c

@@ -11,6 +11,9 @@
 #include "nvim/msgpack_rpc/channel.h"
 #include "nvim/msgpack_rpc/server.h"
 #include "nvim/os/shell.h"
+#ifdef WIN32
+# include "nvim/os/os_win_conpty.h"
+#endif
 #include "nvim/path.h"
 #include "nvim/ascii.h"
 
@@ -469,8 +472,34 @@ uint64_t channel_from_stdio(bool rpc, CallbackReader on_output,
 
   Channel *channel = channel_alloc(kChannelStreamStdio);
 
-  rstream_init_fd(&main_loop, &channel->stream.stdio.in, 0, 0);
-  wstream_init_fd(&main_loop, &channel->stream.stdio.out, 1, 0);
+  int stdin_dup_fd = STDIN_FILENO;
+  int stdout_dup_fd = STDOUT_FILENO;
+#ifdef WIN32
+  // Strangely, ConPTY doesn't work if stdin and stdout are pipes. So replace
+  // stdin and stdout with CONIN$ and CONOUT$, respectively.
+  if (embedded_mode && os_has_conpty_working()) {
+    stdin_dup_fd = os_dup(STDIN_FILENO);
+    close(STDIN_FILENO);
+    const HANDLE conin_handle =
+      CreateFile("CONIN$", GENERIC_READ | GENERIC_WRITE,
+                 FILE_SHARE_READ, (LPSECURITY_ATTRIBUTES)NULL,
+                 OPEN_EXISTING, 0, (HANDLE)NULL);
+    assert(conin_handle != INVALID_HANDLE_VALUE);
+    const int conin_fd = _open_osfhandle((intptr_t)conin_handle, _O_RDONLY);
+    assert(conin_fd == STDIN_FILENO);
+    stdout_dup_fd = os_dup(STDOUT_FILENO);
+    close(STDOUT_FILENO);
+    const HANDLE conout_handle =
+      CreateFile("CONOUT$", GENERIC_READ | GENERIC_WRITE,
+                 FILE_SHARE_READ, (LPSECURITY_ATTRIBUTES)NULL,
+                 OPEN_EXISTING, 0, (HANDLE)NULL);
+    assert(conout_handle != INVALID_HANDLE_VALUE);
+    const int conout_fd = _open_osfhandle((intptr_t)conout_handle, 0);
+    assert(conout_fd == STDOUT_FILENO);
+  }
+#endif
+  rstream_init_fd(&main_loop, &channel->stream.stdio.in, stdin_dup_fd, 0);
+  wstream_init_fd(&main_loop, &channel->stream.stdio.out, stdout_dup_fd, 0);
 
   if (rpc) {
     rpc_start(channel);

src/nvim/main.c

@@ -46,6 +46,9 @@
 #include "nvim/option.h"
 #include "nvim/os_unix.h"
 #include "nvim/os/os_defs.h"
+#ifdef WIN32
+# include "nvim/os/os_win_conpty.h"
+#endif
 #include "nvim/path.h"
 #include "nvim/profile.h"
 #include "nvim/popupmnu.h"
@@ -226,6 +229,9 @@ void early_init(void)
 
   init_signs();
   ui_comp_syn_init();
+#ifdef WIN32
+  os_dyn_conpty_init();
+#endif
 }
 
 #ifdef MAKE_LIB

src/nvim/option.c

@@ -76,6 +76,9 @@
 #include "nvim/undo.h"
 #include "nvim/window.h"
 #include "nvim/os/os.h"
+#ifdef WIN32
+# include "nvim/os/os_win_conpty.h"
+#endif
 #include "nvim/api/private/helpers.h"
 #include "nvim/os/input.h"
 #include "nvim/os/lang.h"
@@ -310,6 +313,9 @@ static char *(p_scl_values[]) =       { "yes", "no", "auto", "auto:1", "auto:2",
   "auto:3", "auto:4", "auto:5", "auto:6", "auto:7", "auto:8", "auto:9",
   "yes:1", "yes:2", "yes:3", "yes:4", "yes:5", "yes:6", "yes:7", "yes:8",
   "yes:9", NULL };
+#ifdef WIN32
+static char *(p_twt_values[]) =       { "conpty", "winpty", "", NULL };
+#endif
 
 /// All possible flags for 'shm'.
 static char_u SHM_ALL[] = {
@@ -3282,6 +3288,13 @@ ambw_end:
     if (!parse_winhl_opt(curwin)) {
       errmsg = e_invarg;
     }
+#ifdef WIN32
+  } else if (varp == &p_twt) {
+    if (check_opt_strings(*varp, p_twt_values, false) != OK
+        || (!os_has_conpty_working() && STRCMP(*varp, "conpty") == 0)) {
+      errmsg = e_invarg;
+    }
+#endif
   } else {
     // Options that are a list of flags.
     p = NULL;

src/nvim/option_defs.h

@@ -654,6 +654,7 @@ EXTERN char_u *p_titleold;      ///< 'titleold'
 EXTERN char_u *p_titlestring;   ///< 'titlestring'
 EXTERN char_u *p_tsr;           ///< 'thesaurus'
 EXTERN int p_tgc;               ///< 'termguicolors'
+EXTERN char_u *p_twt;           ///< 'termwintype'
 EXTERN int p_ttimeout;          ///< 'ttimeout'
 EXTERN long p_ttm;              ///< 'ttimeoutlen'
 EXTERN char_u *p_udir;          ///< 'undodir'

src/nvim/options.lua

@@ -2496,6 +2496,13 @@ return {
       varname='p_tgc',
       defaults={if_true={vi=false}}
     },
+    {
+      full_name='termwintype', abbreviation='twt',
+      type='string', scope={'global'},
+      vi_def=false,
+      varname='p_twt',
+      defaults={if_true={vi="",vim=""}}
+    },
     {
       full_name='terse',
       type='bool', scope={'global'},

src/nvim/os/os_win_conpty.c

@@ -0,0 +1,181 @@
+#include <uv.h>
+
+#include "nvim/vim.h"
+#include "nvim/os/os.h"
+#include "nvim/os/os_win_conpty.h"
+
+#ifndef EXTENDED_STARTUPINFO_PRESENT
+# define EXTENDED_STARTUPINFO_PRESENT 0x00080000
+#endif
+#ifndef PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE
+# define PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE 0x00020016
+#endif
+
+static bool conpty_working = false;
+
+HRESULT (WINAPI *pCreatePseudoConsole)(COORD, HANDLE, HANDLE, DWORD, HPCON *);
+HRESULT (WINAPI *pResizePseudoConsole)(HPCON, COORD);
+HRESULT (WINAPI *pClosePseudoConsole)(HPCON);
+
+bool os_has_conpty_working(void)
+{
+  return conpty_working;
+}
+
+void os_dyn_conpty_init(void)
+{
+  uv_lib_t kernel;
+  if (uv_dlopen("kernel32.dll", &kernel)) {
+    uv_dlclose(&kernel);
+    return;
+  }
+  static struct {
+    char *name;
+    FARPROC *ptr;
+  } conpty_entry[] = {
+    { "CreatePseudoConsole", (FARPROC *)&pCreatePseudoConsole },
+    { "ResizePseudoConsole", (FARPROC *)&pResizePseudoConsole },
+    { "ClosePseudoConsole", (FARPROC *)&pClosePseudoConsole },
+    { NULL, NULL }
+  };
+  for (int i = 0;
+       conpty_entry[i].name != NULL && conpty_entry[i].ptr != NULL; i++) {
+    if (uv_dlsym(&kernel, conpty_entry[i].name, (void **)conpty_entry[i].ptr)) {
+      uv_dlclose(&kernel);
+      return;
+    }
+  }
+  conpty_working = true;
+}
+
+conpty_t *os_conpty_init(char **in_name, char **out_name,
+                         uint16_t width, uint16_t height)
+{
+  static int count = 0;
+  conpty_t *conpty_object = xcalloc(1, sizeof(*conpty_object));
+  const char *emsg = NULL;
+  HANDLE in_read = INVALID_HANDLE_VALUE;
+  HANDLE out_write = INVALID_HANDLE_VALUE;
+  char buf[MAXPATHL];
+  SECURITY_ATTRIBUTES sa = { 0 };
+  const DWORD mode = PIPE_ACCESS_INBOUND
+    | PIPE_ACCESS_OUTBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE;
+
+  sa.nLength = sizeof(sa);
+  snprintf(buf, sizeof(buf), "\\\\.\\pipe\\nvim-term-in-%d-%d",
+           os_get_pid(), count);
+  *in_name = xstrdup(buf);
+  if ((in_read = CreateNamedPipeA(
+      *in_name,
+      mode,
+      PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT,
+      1,
+      0,
+      0,
+      30000,
+      &sa)) == INVALID_HANDLE_VALUE) {
+    emsg = "create input pipe failed";
+    goto failed;
+  }
+  snprintf(buf, sizeof(buf), "\\\\.\\pipe\\nvim-term-out-%d-%d",
+           os_get_pid(), count);
+  *out_name = xstrdup(buf);
+  if ((out_write = CreateNamedPipeA(
+      *out_name,
+      mode,
+      PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT,
+      1,
+      0,
+      0,
+      30000,
+      &sa)) == INVALID_HANDLE_VALUE) {
+    emsg = "create output pipe failed";
+    goto failed;
+  }
+  assert(width <= SHRT_MAX);
+  assert(height <=  SHRT_MAX);
+  COORD size = { (int16_t)width, (int16_t)height };
+  HRESULT hr;
+  hr = pCreatePseudoConsole(size, in_read, out_write, 0, &conpty_object->pty);
+  if (FAILED(hr)) {
+    emsg = "create psudo console failed";
+    goto failed;
+  }
+
+  conpty_object->si_ex.StartupInfo.cb = sizeof(conpty_object->si_ex);
+  size_t bytes_required;
+  InitializeProcThreadAttributeList(NULL, 1, 0,  & bytes_required);
+  conpty_object->si_ex.lpAttributeList =
+    (PPROC_THREAD_ATTRIBUTE_LIST)xmalloc(bytes_required);
+  if (!InitializeProcThreadAttributeList(
+      conpty_object->si_ex.lpAttributeList,
+      1,
+      0,
+      &bytes_required)) {
+    emsg = "InitializeProcThreadAttributeList failed";
+    goto failed;
+  }
+  if (!UpdateProcThreadAttribute(
+      conpty_object->si_ex.lpAttributeList,
+      0,
+      PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE,
+      conpty_object->pty,
+      sizeof(conpty_object->pty),
+      NULL,
+      NULL)) {
+    emsg = "UpdateProcThreadAttribute failed";
+    goto failed;
+  }
+  count++;
+  goto finished;
+
+failed:
+  ELOG("os_conpty_init:%s : error code: %d",
+       emsg, os_translate_sys_error((int)GetLastError()));
+  os_conpty_free(conpty_object);
+  conpty_object = NULL;
+finished:
+  if (in_read != INVALID_HANDLE_VALUE) {
+    CloseHandle(in_read);
+  }
+  if (out_write != INVALID_HANDLE_VALUE) {
+    CloseHandle(out_write);
+  }
+  return conpty_object;
+}
+
+bool os_conpty_spawn(conpty_t *conpty_object, HANDLE *process_handle,
+                     wchar_t *name, wchar_t *cmd_line, wchar_t *cwd,
+                     wchar_t *env)
+{
+  PROCESS_INFORMATION pi = { 0 };
+  if (!CreateProcessW(
+      name,
+      cmd_line,
+      NULL,
+      NULL,
+      false,
+      EXTENDED_STARTUPINFO_PRESENT | CREATE_UNICODE_ENVIRONMENT,
+      env,
+      cwd,
+      &conpty_object->si_ex.StartupInfo,
+      &pi)) {
+    return false;
+  }
+  *process_handle = pi.hProcess;
+  return true;
+}
+
+void os_conpty_free(conpty_t * conpty_object)
+{
+  if (conpty_object != NULL) {
+    if (conpty_object->si_ex.lpAttributeList != NULL) {
+      DeleteProcThreadAttributeList(conpty_object->si_ex.lpAttributeList);
+      xfree(conpty_object->si_ex.lpAttributeList);
+    }
+    if (conpty_object->pty != NULL) {
+      pClosePseudoConsole(conpty_object->pty);
+    }
+  }
+  xfree(conpty_object);
+}

src/nvim/os/os_win_conpty.h

@@ -0,0 +1,22 @@
+#ifndef NVIM_OS_OS_WIN_CONPTY_H
+#define NVIM_OS_OS_WIN_CONPTY_H
+
+#ifndef HPCON
+# define HPCON VOID *
+#endif
+
+extern HRESULT (WINAPI *pCreatePseudoConsole)  // NOLINT(whitespace/parens)
+  (COORD, HANDLE, HANDLE, DWORD, HPCON *);
+extern HRESULT (WINAPI *pResizePseudoConsole)(HPCON, COORD);
+extern HRESULT (WINAPI *pClosePseudoConsole)(HPCON);
+
+typedef struct conpty {
+  HPCON pty;
+  STARTUPINFOEXW si_ex;
+} conpty_t;
+
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "os/os_win_conpty.h.generated.h"
+#endif
+
+#endif  // NVIM_OS_OS_WIN_CONPTY_H

src/nvim/os/pty_process_win.c

@@ -12,6 +12,7 @@
 #include "nvim/memory.h"
 #include "nvim/mbyte.h"  // for utf8_to_utf16, utf16_to_utf8
 #include "nvim/os/pty_process_win.h"
+#include "nvim/os/os_win_conpty.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "os/pty_process_win.c.generated.h"
@@ -23,6 +24,11 @@ static void CALLBACK pty_process_finish1(void *context, BOOLEAN unused)
   PtyProcess *ptyproc = (PtyProcess *)context;
   Process *proc = (Process *)ptyproc;
 
+  if (ptyproc->type == PTY_TYPE_CONPTY
+      && ptyproc->pty_object.conpty_object != NULL) {
+    os_conpty_free(ptyproc->pty_object.conpty_object);
+    ptyproc->pty_object.conpty_object = NULL;
+  }
   uv_timer_init(&proc->loop->uv, &ptyproc->wait_eof_timer);
   ptyproc->wait_eof_timer.data = (void *)ptyproc;
   uv_timer_start(&ptyproc->wait_eof_timer, wait_eof_timer_cb, 200, 200);
@@ -38,6 +44,7 @@ int pty_process_spawn(PtyProcess *ptyproc)
   winpty_config_t *cfg = NULL;
   winpty_spawn_config_t *spawncfg = NULL;
   winpty_t *winpty_object = NULL;
+  conpty_t *conpty_object = NULL;
   char *in_name = NULL;
   char *out_name = NULL;
   HANDLE process_handle = NULL;
@@ -49,6 +56,16 @@ int pty_process_spawn(PtyProcess *ptyproc)
 
   assert(proc->err.closed);
 
+  int pty_type = *p_twt;
+  if (pty_type == 'c' && os_has_conpty_working()) {
+    if ((conpty_object =
+         os_conpty_init(&in_name, &out_name,
+                        ptyproc->width, ptyproc->height)) != NULL) {
+      ptyproc->type = PTY_TYPE_CONPTY;
+    }
+  }
+
+  if (ptyproc->type == PTY_TYPE_WINPTY) {
     cfg = winpty_config_new(WINPTY_FLAG_ALLOW_CURPROC_DESKTOP_CREATION, &err);
     if (cfg == NULL) {
       emsg = "winpty_config_new failed";
@@ -73,6 +90,7 @@ int pty_process_spawn(PtyProcess *ptyproc)
       emsg = "utf16_to_utf8(winpty_conout_name) failed";
       goto cleanup;
     }
+  }
 
   if (!proc->in.closed) {
     in_req = xmalloc(sizeof(uv_connect_t));
@@ -107,6 +125,18 @@ int pty_process_spawn(PtyProcess *ptyproc)
     goto cleanup;
   }
 
+  if (ptyproc->type == PTY_TYPE_CONPTY) {
+    if (!os_conpty_spawn(conpty_object,
+                         &process_handle,
+                         NULL,
+                         cmd_line,
+                         cwd,
+                         NULL)) {
+      emsg = "os_conpty_spawn failed";
+      status = (int)GetLastError();
+      goto cleanup;
+    }
+  } else {
     spawncfg = winpty_spawn_config_new(
         WINPTY_SPAWN_FLAG_AUTO_SHUTDOWN,
         NULL,  // Optional application name
@@ -134,6 +164,7 @@ int pty_process_spawn(PtyProcess *ptyproc)
       }
       goto cleanup;
     }
+  }
   proc->pid = (int)GetProcessId(process_handle);
 
   if (!RegisterWaitForSingleObject(
@@ -152,9 +183,14 @@ int pty_process_spawn(PtyProcess *ptyproc)
     uv_run(&proc->loop->uv, UV_RUN_ONCE);
   }
 
-  ptyproc->winpty_object = winpty_object;
+  if (ptyproc->type == PTY_TYPE_CONPTY) {
+    ptyproc->pty_object.conpty_object = conpty_object;
+  } else {
+    ptyproc->pty_object.winpty_object = winpty_object;
+  }
   ptyproc->process_handle = process_handle;
   winpty_object = NULL;
+  conpty_object = NULL;
   process_handle = NULL;
 
 cleanup:
@@ -171,6 +207,7 @@ cleanup:
   winpty_config_free(cfg);
   winpty_spawn_config_free(spawncfg);
   winpty_free(winpty_object);
+  os_conpty_free(conpty_object);
   xfree(in_name);
   xfree(out_name);
   if (process_handle != NULL) {
@@ -192,8 +229,18 @@ void pty_process_resize(PtyProcess *ptyproc, uint16_t width,
                         uint16_t height)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (ptyproc->winpty_object != NULL) {
-    winpty_set_size(ptyproc->winpty_object, width, height, NULL);
+  if (ptyproc->type == PTY_TYPE_CONPTY
+      && ptyproc->pty_object.conpty_object != NULL) {
+    assert(width <= SHRT_MAX);
+    assert(height <= SHRT_MAX);
+    COORD size = { (int16_t)width, (int16_t)height };
+    if (pResizePseudoConsole(
+        ptyproc->pty_object.conpty_object->pty, size) != S_OK) {
+      ELOG("ResizePseudoConsoel failed: error code: %d",
+           os_translate_sys_error((int)GetLastError()));
+    }
+  } else if (ptyproc->pty_object.winpty_object != NULL) {
+    winpty_set_size(ptyproc->pty_object.winpty_object, width, height, NULL);
   }
 }
 
@@ -212,9 +259,10 @@ void pty_process_close(PtyProcess *ptyproc)
 void pty_process_close_master(PtyProcess *ptyproc)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (ptyproc->winpty_object != NULL) {
-    winpty_free(ptyproc->winpty_object);
-    ptyproc->winpty_object = NULL;
+  if (ptyproc->type == PTY_TYPE_WINPTY
+      && ptyproc->pty_object.winpty_object != NULL) {
+    winpty_free(ptyproc->pty_object.winpty_object);
+    ptyproc->pty_object.winpty_object = NULL;
   }
 }
 

src/nvim/os/pty_process_win.h

@@ -6,12 +6,22 @@
 
 #include "nvim/event/process.h"
 #include "nvim/lib/queue.h"
+#include "nvim/os/os_win_conpty.h"
+
+typedef enum {
+  PTY_TYPE_WINPTY,
+  PTY_TYPE_CONPTY
+} pty_type_t;
 
 typedef struct pty_process {
   Process process;
   char *term_name;
   uint16_t width, height;
+  union {
     winpty_t *winpty_object;
+    conpty_t *conpty_object;
+  } pty_object;
+  pty_type_t type;
   HANDLE finish_wait;
   HANDLE process_handle;
   uv_timer_t wait_eof_timer;
@@ -30,7 +40,8 @@ static inline PtyProcess pty_process_init(Loop *loop, void *data)
   rv.term_name = NULL;
   rv.width = 80;
   rv.height = 24;
-  rv.winpty_object = NULL;
+  rv.pty_object.winpty_object = NULL;
+  rv.type = PTY_TYPE_WINPTY;
   rv.finish_wait = NULL;
   rv.process_handle = NULL;
   return rv;