perf(highlight): allow decoration providers to skip ranges without data

Continuing the work of #31400

That PR allowed the provider to be invoked multiple times per line.
We want only to do that when there actually is more data later on the
line. Additionally, we want to skip over lines which contain no new
highlight items. The TS query cursor already tells us what the next
position with more data is, so there is no need to reinvoke the range
callback before that.

NB: this removes the double buffering introduced in #32619 which
is funtamentally incompatible with this (nvim core is supposed to keep
track of long ranges by itself, without requiring a callback reinvoke
blitz). Need to adjust the priorities some other way to fix the same issue.

src/nvim/lua/executor.c

@@ -181,12 +181,9 @@ int nlua_pcall(lua_State *lstate, int nargs, int nresults)
     lua_remove(lstate, -2);
   } else {
     if (nresults == LUA_MULTRET) {
-      int new_top = lua_gettop(lstate);
-      int actual_nres = new_top - pre_top + nargs + 1;
-      lua_remove(lstate, -1 - actual_nres);
-    } else {
-      lua_remove(lstate, -1 - nresults);
+      nresults = lua_gettop(lstate) - (pre_top - nargs - 1);
     }
+    lua_remove(lstate, -1 - nresults);
   }
   return status;
 }
@@ -1551,6 +1548,7 @@ Object nlua_exec(const String str, const char *chunkname, const Array args, LuaR
 {
   lua_State *const lstate = global_lstate;
 
+  int top = lua_gettop(lstate);
   const char *name = (chunkname && chunkname[0]) ? chunkname : "<nvim>";
   if (luaL_loadbuffer(lstate, str.data, str.size, name)) {
     size_t len;
@@ -1570,7 +1568,7 @@ Object nlua_exec(const String str, const char *chunkname, const Array args, LuaR
     return NIL;
   }
 
-  return nlua_call_pop_retval(lstate, mode, arena, err);
+  return nlua_call_pop_retval(lstate, mode, arena, top, err);
 }
 
 bool nlua_ref_is_function(LuaRef ref)
@@ -1601,10 +1599,16 @@ Object nlua_call_ref(LuaRef ref, const char *name, Array args, LuaRetMode mode,
   return nlua_call_ref_ctx(false, ref, name, args, mode, arena, err);
 }
 
+static int mode_ret(LuaRetMode mode)
+{
+  return mode == kRetMulti ? LUA_MULTRET : 1;
+}
+
 Object nlua_call_ref_ctx(bool fast, LuaRef ref, const char *name, Array args, LuaRetMode mode,
                          Arena *arena, Error *err)
 {
   lua_State *const lstate = global_lstate;
+  int top = lua_gettop(lstate);
   nlua_pushref(lstate, ref);
   int nargs = (int)args.size;
   if (name != NULL) {
@@ -1616,12 +1620,12 @@ Object nlua_call_ref_ctx(bool fast, LuaRef ref, const char *name, Array args, Lu
   }
 
   if (fast) {
-    if (nlua_fast_cfpcall(lstate, nargs, 1, -1) < 0) {
+    if (nlua_fast_cfpcall(lstate, nargs, mode_ret(mode), -1) < 0) {
       // error is already scheduled, set anyways to convey failure.
       api_set_error(err, kErrorTypeException, "fast context failure");
       return NIL;
     }
-  } else if (nlua_pcall(lstate, nargs, 1)) {
+  } else if (nlua_pcall(lstate, nargs, mode_ret(mode))) {
     // if err is passed, the caller will deal with the error.
     if (err) {
       size_t len;
@@ -1633,16 +1637,18 @@ Object nlua_call_ref_ctx(bool fast, LuaRef ref, const char *name, Array args, Lu
     return NIL;
   }
 
-  return nlua_call_pop_retval(lstate, mode, arena, err);
+  return nlua_call_pop_retval(lstate, mode, arena, top, err);
 }
 
-static Object nlua_call_pop_retval(lua_State *lstate, LuaRetMode mode, Arena *arena, Error *err)
+static Object nlua_call_pop_retval(lua_State *lstate, LuaRetMode mode, Arena *arena, int pretop,
+                                   Error *err)
 {
-  if (lua_isnil(lstate, -1)) {
+  if (mode != kRetMulti && lua_isnil(lstate, -1)) {
     lua_pop(lstate, 1);
     return NIL;
   }
   Error dummy = ERROR_INIT;
+  Error *perr = err ? err : &dummy;
 
   switch (mode) {
   case kRetNilBool: {
@@ -1658,7 +1664,19 @@ static Object nlua_call_pop_retval(lua_State *lstate, LuaRetMode mode, Arena *ar
     return LUAREF_OBJ(ref);
   }
   case kRetObject:
-    return nlua_pop_Object(lstate, false, arena, err ? err : &dummy);
+    return nlua_pop_Object(lstate, false, arena, perr);
+  case kRetMulti:
+    ;
+    int nres = lua_gettop(lstate) - pretop;
+    Array res = arena_array(arena, (size_t)nres);
+    for (int i = 0; i < nres; i++) {
+      res.items[nres - i - 1] = nlua_pop_Object(lstate, false, arena, perr);
+      if (ERROR_SET(err)) {
+        return NIL;
+      }
+    }
+    res.size = (size_t)nres;
+    return ARRAY_OBJ(res);
   }
   UNREACHABLE;
 }

src/nvim/lua/executor.h

@@ -38,10 +38,11 @@ typedef struct {
   } while (0)
 
 typedef enum {
-  kRetObject,  ///< any object, but doesn't preserve nested luarefs
+  kRetObject,   ///< any object, but doesn't preserve nested luarefs
   kRetNilBool,  ///< NIL preserved as such, other values return their booleanness
                 ///< Should also be used when return value is ignored, as it is allocation-free
-  kRetLuaref,  ///< return value becomes a single Luaref, regardless of type (except NIL)
+  kRetLuaref,   ///< return value becomes a single Luaref, regardless of type (except NIL)
+  kRetMulti,    ///< like kRetObject but return muliple return values as an Array
 } LuaRetMode;
 
 /// Maximum number of errors in vim.ui_attach() and decor provider callbacks.