job control: reuse common job code for rpc jobs

This makes stderr and exit callbacks work for rpc jobs

src/nvim/eval.c

@@ -408,6 +408,7 @@ typedef struct {
   Terminal *term;
   bool stopped;
   bool exited;
+  bool rpc;
   int refcount;
   ufunc_T *on_stdout, *on_stderr, *on_exit;
   dict_T *self;
@@ -448,8 +449,7 @@ typedef struct {
 #define FNE_INCL_BR     1       /* find_name_end(): include [] in name */
 #define FNE_CHECK_START 2       /* find_name_end(): check name starts with
                                    valid character */
-static uint64_t current_job_id = 1;
-static PMap(uint64_t) *jobs = NULL; 
+static PMap(uint64_t) *jobs = NULL;
 
 static uint64_t last_timer_id = 0;
 static PMap(uint64_t) *timers = NULL;
@@ -11724,16 +11724,35 @@ static void f_jobclose(typval_T *argvars, typval_T *rettv)
   if (argvars[1].v_type == VAR_STRING) {
     char *stream = (char *)argvars[1].vval.v_string;
     if (!strcmp(stream, "stdin")) {
-      process_close_in(proc);
+      if (data->rpc) {
+        EMSG(_("Invalid stream on rpc job, use jobclose(id, 'rpc')"));
+      } else {
+        process_close_in(proc);
+      }
     } else if (!strcmp(stream, "stdout")) {
-      process_close_out(proc);
+      if (data->rpc) {
+        EMSG(_("Invalid stream on rpc job, use jobclose(id, 'rpc')"));
+      } else {
+        process_close_out(proc);
+      }
     } else if (!strcmp(stream, "stderr")) {
       process_close_err(proc);
+    } else if (!strcmp(stream, "rpc")) {
+      if (data->rpc) {
+        channel_close(data->id);
+      } else {
+        EMSG(_("Invalid job stream: Not an rpc job"));
+      }
     } else {
       EMSG2(_("Invalid job stream \"%s\""), stream);
     }
   } else {
-    process_close_streams(proc);
+    if (data->rpc) {
+      channel_close(data->id);
+      process_close_err(proc);
+    } else {
+      process_close_streams(proc);
+    }
   }
 }
 
@@ -11790,6 +11809,11 @@ static void f_jobsend(typval_T *argvars, typval_T *rettv)
     return;
   }
 
+  if (data->rpc) {
+    EMSG(_("Can't send raw data to rpc channel"));
+    return;
+  }
+
   ssize_t input_len;
   char *input = (char *) save_tv_as_string(&argvars[1], &input_len, false);
   if (!input) {
@@ -11911,12 +11935,23 @@ static void f_jobstart(typval_T *argvars, typval_T *rettv)
     return;
   }
 
+
   dict_T *job_opts = NULL;
+  bool detach = false, rpc = false, pty = false;
   ufunc_T *on_stdout = NULL, *on_stderr = NULL, *on_exit = NULL;
   char *cwd = NULL;
   if (argvars[1].v_type == VAR_DICT) {
     job_opts = argvars[1].vval.v_dict;
 
+    detach = get_dict_number(job_opts, (uint8_t *)"detach") != 0;
+    rpc = get_dict_number(job_opts, (uint8_t *)"rpc") != 0;
+    pty = get_dict_number(job_opts, (uint8_t *)"pty") != 0;
+    if (pty && rpc) {
+      EMSG2(_(e_invarg2), "job cannot have both 'pty' and 'rpc' options set");
+      shell_free_argv(argv);
+      return;
+    }
+
     char *new_cwd = (char *)get_dict_string(job_opts, (char_u *)"cwd", false);
     if (new_cwd && strlen(new_cwd) > 0) {
       cwd = new_cwd;
@@ -11934,10 +11969,8 @@ static void f_jobstart(typval_T *argvars, typval_T *rettv)
     }
   }
 
-  bool pty = job_opts && get_dict_number(job_opts, (uint8_t *)"pty") != 0;
-  bool detach = job_opts && get_dict_number(job_opts, (uint8_t *)"detach") != 0;
   TerminalJobData *data = common_job_init(argv, on_stdout, on_stderr, on_exit,
-                                          job_opts, pty, detach, cwd);
+                                          job_opts, pty, rpc, detach, cwd);
   Process *proc = (Process *)&data->proc;
 
   if (pty) {
@@ -11955,7 +11988,7 @@ static void f_jobstart(typval_T *argvars, typval_T *rettv)
     }
   }
 
-  if (!on_stdout) {
+  if (!rpc && !on_stdout) {
     proc->out = NULL;
   }
   if (!on_stderr) {
@@ -14105,7 +14138,7 @@ end:
   api_free_object(result);
 }
 
-// "rpcstart()" function
+// "rpcstart()" function (DEPRECATED)
 static void f_rpcstart(typval_T *argvars, typval_T *rettv)
 {
   rettv->v_type = VAR_NUMBER;
@@ -14158,32 +14191,27 @@ static void f_rpcstart(typval_T *argvars, typval_T *rettv)
 
   // The last item of argv must be NULL
   argv[i] = NULL;
-  uint64_t channel_id = channel_from_process(argv);
 
-  if (!channel_id) {
-    EMSG(_(e_api_spawn_failed));
-  }
-
-  rettv->vval.v_number = (varnumber_T)channel_id;
+  TerminalJobData *data = common_job_init(argv, NULL, NULL, NULL,
+                                          NULL, false, true, false, NULL);
+  common_job_start(data, rettv);
 }
 
 // "rpcstop()" function
 static void f_rpcstop(typval_T *argvars, typval_T *rettv)
 {
-  rettv->v_type = VAR_NUMBER;
-  rettv->vval.v_number = 0;
-
-  if (check_restricted() || check_secure()) {
-    return;
-  }
-
   if (argvars[0].v_type != VAR_NUMBER) {
     // Wrong argument types
     EMSG(_(e_invarg));
     return;
   }
 
-  rettv->vval.v_number = channel_close(argvars[0].vval.v_number);
+  // if called with a job, stop it, else closes the channel
+  if (pmap_get(uint64_t)(jobs, argvars[0].vval.v_number)) {
+    f_jobstop(argvars, rettv);
+  } else {
+    rettv->vval.v_number = channel_close(argvars[0].vval.v_number);
+  }
 }
 
 /*
@@ -16677,7 +16705,7 @@ static void f_termopen(typval_T *argvars, typval_T *rettv)
   }
 
   TerminalJobData *data = common_job_init(argv, on_stdout, on_stderr, on_exit,
-                                          job_opts, true, false, cwd);
+                                          job_opts, true, false, false, cwd);
   data->proc.pty.width = curwin->w_width;
   data->proc.pty.height = curwin->w_height;
   data->proc.pty.term_name = xstrdup("xterm-256color");
@@ -22101,6 +22129,7 @@ static inline TerminalJobData *common_job_init(char **argv,
                                                ufunc_T *on_exit,
                                                dict_T *self,
                                                bool pty,
+                                               bool rpc,
                                                bool detach,
                                                char *cwd)
 {
@@ -22111,6 +22140,7 @@ static inline TerminalJobData *common_job_init(char **argv,
   data->on_exit = on_exit;
   data->self = self;
   data->events = queue_new_child(main_loop.events);
+  data->rpc = rpc;
   if (pty) {
     data->proc.pty = pty_process_init(&main_loop, data);
   } else {
@@ -22130,7 +22160,9 @@ static inline TerminalJobData *common_job_init(char **argv,
   return data;
 }
 
-/// Return true/false on success/failure.
+/// common code for getting job callbacks for jobstart, termopen and rpcstart
+///
+/// @return true/false on success/failure.
 static inline bool common_job_callbacks(dict_T *vopts, ufunc_T **on_stdout,
                                         ufunc_T **on_stderr, ufunc_T **on_exit)
 {
@@ -22174,12 +22206,19 @@ static inline bool common_job_start(TerminalJobData *data, typval_T *rettv)
   }
   xfree(cmd);
 
-  data->id = current_job_id++;
-  wstream_init(proc->in, 0);
-  if (proc->out) {
-    rstream_init(proc->out, 0);
-    rstream_start(proc->out, on_job_stdout, data);
+  data->id = next_chan_id++;
+
+  if (data->rpc) {
+    // the rpc channel takes over the in and out streams
+    channel_from_process(proc, data->id);
+  } else {
+    wstream_init(proc->in, 0);
+    if (proc->out) {
+      rstream_init(proc->out, 0);
+      rstream_start(proc->out, on_job_stdout, data);
+    }
   }
+
   if (proc->err) {
     rstream_init(proc->err, 0);
     rstream_start(proc->err, on_job_stderr, data);
@@ -22302,12 +22341,18 @@ static void on_process_exit(Process *proc, int status, void *d)
     snprintf(msg, sizeof msg, "\r\n[Process exited %d]", proc->status);
     terminal_close(data->term, msg);
   }
+  if (data->rpc) {
+    channel_process_exit(data->id, status);
+  }
 
   if (data->status_ptr) {
     *data->status_ptr = status;
   }
 
   process_job_event(data, data->on_exit, "exit", NULL, 0, status);
+
+  pmap_del(uint64_t)(jobs, data->id);
+  term_job_data_decref(data);
 }
 
 static void term_write(char *buf, size_t size, void *d)
@@ -22355,7 +22400,7 @@ static void term_job_data_decref(TerminalJobData *data)
 static void on_job_event(JobEvent *ev)
 {
   if (!ev->callback) {
-    goto end;
+    return;
   }
 
   typval_T argv[3];
@@ -22391,13 +22436,6 @@ static void on_job_event(JobEvent *ev)
   call_user_func(ev->callback, argc, argv, &rettv, curwin->w_cursor.lnum,
       curwin->w_cursor.lnum, ev->data->self);
   clear_tv(&rettv);
-
-end:
-  if (!ev->received) {
-    // exit event, safe to free job data now
-    pmap_del(uint64_t)(jobs, ev->data->id);
-    term_job_data_decref(ev->data);
-  }
 }
 
 static TerminalJobData *find_job(uint64_t id)

src/nvim/globals.h

@@ -1244,6 +1244,9 @@ EXTERN char *ignoredp;
 // If a msgpack-rpc channel should be started over stdin/stdout
 EXTERN bool embedded_mode INIT(= false);
 
+/// next free id for a job or rpc channel
+EXTERN uint64_t next_chan_id INIT(= 1);
+
 /// Used to track the status of external functions.
 /// Currently only used for iconv().
 typedef enum {

src/nvim/msgpack_rpc/channel.c

@@ -19,6 +19,7 @@
 #include "nvim/main.h"
 #include "nvim/ascii.h"
 #include "nvim/memory.h"
+#include "nvim/eval.h"
 #include "nvim/os_unix.h"
 #include "nvim/message.h"
 #include "nvim/map.h"
@@ -55,12 +56,7 @@ typedef struct {
   msgpack_unpacker *unpacker;
   union {
     Stream stream;
-    struct {
-      LibuvProcess uvproc;
-      Stream in;
-      Stream out;
-      Stream err;
-    } process;
+    Process *proc;
     struct {
       Stream in;
       Stream out;
@@ -79,7 +75,6 @@ typedef struct {
   uint64_t request_id;
 } RequestEvent;
 
-static uint64_t next_id = 1;
 static PMap(uint64_t) *channels = NULL;
 static PMap(cstr_t) *event_strings = NULL;
 static msgpack_sbuffer out_buffer;
@@ -112,33 +107,20 @@ void channel_teardown(void)
 }
 
 /// Creates an API channel by starting a process and connecting to its
-/// stdin/stdout. stderr is forwarded to the editor error stream.
+/// stdin/stdout. stderr is handled by the job infrastructure.
 ///
 /// @param argv The argument vector for the process. [consumed]
 /// @return The channel id (> 0), on success.
 ///         0, on error.
-uint64_t channel_from_process(char **argv)
+uint64_t channel_from_process(Process *proc, uint64_t id)
 {
-  Channel *channel = register_channel(kChannelTypeProc);
-  channel->data.process.uvproc = libuv_process_init(&main_loop, channel);
-  Process *proc = &channel->data.process.uvproc.process;
-  proc->argv = argv;
-  proc->in = &channel->data.process.in;
-  proc->out = &channel->data.process.out;
-  proc->err = &channel->data.process.err;
-  proc->cb = process_exit;
-  if (!process_spawn(proc)) {
-    loop_poll_events(&main_loop, 0);
-    decref(channel);
-    return 0;
-  }
-
+  Channel *channel = register_channel(kChannelTypeProc, id, proc->events);
   incref(channel);  // process channels are only closed by the exit_cb
+  channel->data.proc = proc;
+
   wstream_init(proc->in, 0);
   rstream_init(proc->out, 0);
   rstream_start(proc->out, parse_msgpack, channel);
-  rstream_init(proc->err, 0);
-  rstream_start(proc->err, forward_stderr, channel);
 
   return channel->id;
 }
@@ -148,7 +130,7 @@ uint64_t channel_from_process(char **argv)
 /// @param watcher The SocketWatcher ready to accept the connection
 void channel_from_connection(SocketWatcher *watcher)
 {
-  Channel *channel = register_channel(kChannelTypeSocket);
+  Channel *channel = register_channel(kChannelTypeSocket, 0, NULL);
   socket_watcher_accept(watcher, &channel->data.stream);
   incref(channel);  // close channel only after the stream is closed
   channel->data.stream.internal_close_cb = close_cb;
@@ -314,7 +296,7 @@ bool channel_close(uint64_t id)
 /// Neovim
 void channel_from_stdio(void)
 {
-  Channel *channel = register_channel(kChannelTypeStdio);
+  Channel *channel = register_channel(kChannelTypeStdio, 0, NULL);
   incref(channel);  // stdio channels are only closed on exit
   // read stream
   rstream_init_fd(&main_loop, &channel->data.std.in, 0, CHANNEL_BUFFER_SIZE);
@@ -323,20 +305,12 @@ void channel_from_stdio(void)
   wstream_init_fd(&main_loop, &channel->data.std.out, 1, 0);
 }
 
-static void forward_stderr(Stream *stream, RBuffer *rbuf, size_t count,
-    void *data, bool eof)
+void channel_process_exit(uint64_t id, int status)
 {
-  while (rbuffer_size(rbuf)) {
-    char buf[256];
-    size_t read = rbuffer_read(rbuf, buf, sizeof(buf) - 1);
-    buf[read] = NUL;
-    ELOG("Channel %" PRIu64 " stderr: %s", ((Channel *)data)->id, buf);
-  }
-}
+  Channel *channel = pmap_get(uint64_t)(channels, id);
 
-static void process_exit(Process *proc, int status, void *data)
-{
-  decref(data);
+  channel->closed = true;
+  decref(channel);
 }
 
 static void parse_msgpack(Stream *stream, RBuffer *rbuf, size_t c, void *data,
@@ -511,7 +485,7 @@ static bool channel_write(Channel *channel, WBuffer *buffer)
       success = wstream_write(&channel->data.stream, buffer);
       break;
     case kChannelTypeProc:
-      success = wstream_write(&channel->data.process.in, buffer);
+      success = wstream_write(channel->data.proc->in, buffer);
       break;
     case kChannelTypeStdio:
       success = wstream_write(&channel->data.std.out, buffer);
@@ -639,9 +613,10 @@ static void close_channel(Channel *channel)
       stream_close(&channel->data.stream, NULL, NULL);
       break;
     case kChannelTypeProc:
-      if (!channel->data.process.uvproc.process.closed) {
-        process_stop(&channel->data.process.uvproc.process);
-      }
+      // Only close the rpc channel part,
+      // there could be an error message on the stderr stream
+      process_close_in(channel->data.proc);
+      process_close_out(channel->data.proc);
       break;
     case kChannelTypeStdio:
       stream_close(&channel->data.std.in, NULL, NULL);
@@ -679,7 +654,9 @@ static void free_channel(Channel *channel)
   pmap_free(cstr_t)(channel->subscribed_events);
   kv_destroy(channel->call_stack);
   kv_destroy(channel->delayed_notifications);
-  queue_free(channel->events);
+  if (channel->type != kChannelTypeProc) {
+    queue_free(channel->events);
+  }
   xfree(channel);
 }
 
@@ -688,15 +665,15 @@ static void close_cb(Stream *stream, void *data)
   decref(data);
 }
 
-static Channel *register_channel(ChannelType type)
+static Channel *register_channel(ChannelType type, uint64_t id, Queue *events)
 {
   Channel *rv = xmalloc(sizeof(Channel));
-  rv->events = queue_new_child(main_loop.events);
+  rv->events = events ? events : queue_new_child(main_loop.events);
   rv->type = type;
   rv->refcount = 1;
   rv->closed = false;
   rv->unpacker = msgpack_unpacker_new(MSGPACK_UNPACKER_INIT_BUFFER_SIZE);
-  rv->id = next_id++;
+  rv->id = id > 0 ? id : next_chan_id++;
   rv->pending_requests = 0;
   rv->subscribed_events = pmap_new(cstr_t)();
   rv->next_request_id = 1;

src/nvim/msgpack_rpc/channel.h

@@ -6,6 +6,7 @@
 
 #include "nvim/api/private/defs.h"
 #include "nvim/event/socket.h"
+#include "nvim/event/process.h"
 #include "nvim/vim.h"
 
 #define METHOD_MAXLEN 512