fix(memfile): avoid potential crash on OOM (#37946)

When running out of memory in a libuv callback, try_to_free_memory()
will call mf_release_all(). In this case, mf_sync() cannot call
os_breakcheck() as it'll run the libuv loop recursively, so check
main_loop.recursive to prevent that.

Also fix another possible problem that a terminal buffer may have a
swapfile when encountering an OOM in e.g. terminal_alloc().

src/nvim/eval/funcs.c

@@ -3611,6 +3611,8 @@ void f_jobstart(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
     const int pid = chan->stream.pty.proc.pid;
     buf_T *const buf = curbuf;
 
+    // Unset 'swapfile' to ensure no swapfile is created.
+    buf->b_p_swf = false;
     // If the buffer isn't loaded, open a memfile here to avoid spurious autocommands
     // from open_buffer() when updating the terminal buffer later.
     if (buf->b_ml.ml_mfp == NULL && ml_open(buf) == FAIL) {
@@ -3646,8 +3648,6 @@ void f_jobstart(typval_T *argvars, typval_T *rettv, EvalFuncData fptr)
 
     // Terminal URI: "term://$CWD//$PID:$CMD"
     snprintf(NameBuff, sizeof(NameBuff), "term://%s//%d:%s", IObuff, pid, cmd);
-    // Unset 'swapfile' to ensure no swapfile is created.
-    buf->b_p_swf = false;
 
     setfname(buf, NameBuff, NULL, true);
     apply_autocmds(EVENT_BUFFILEPOST, NULL, NULL, false, buf);

src/nvim/memfile.c

@@ -50,6 +50,7 @@
 #include "nvim/fileio.h"
 #include "nvim/gettext_defs.h"
 #include "nvim/globals.h"
+#include "nvim/main.h"
 #include "nvim/map_defs.h"
 #include "nvim/memfile.h"
 #include "nvim/memfile_defs.h"
@@ -411,7 +412,7 @@ int mf_sync(memfile_T *mfp, int flags)
         if (os_char_avail()) {
           break;
         }
-      } else {
+      } else if (!main_loop.recursive) {  // May reach here on OOM in libuv callback
         os_breakcheck();
       }
       if (got_int) {