api: multiple decoration providers at once

runtime/lua/vim/treesitter/highlighter.lua

@@ -6,6 +6,8 @@ TSHighlighter.__index = TSHighlighter
 
 TSHighlighter.active = TSHighlighter.active or {}
 
+local ns = a.nvim_create_namespace("treesitter/highlighter")
+
 -- These are conventions defined by tree-sitter, though it
 -- needs to be user extensible also.
 TSHighlighter.hl_map = {
@@ -158,7 +160,11 @@ function TSHighlighter._on_line(_, _win, buf, line)
     local start_row, start_col, end_row, end_col = node:range()
     local hl = self.hl_cache[capture]
     if hl and end_row >= line then
-      a.nvim__put_attr(start_row, start_col, { end_line = end_row, end_col = end_col, hl_group = hl })
+      a.nvim_buf_set_extmark(buf, ns, start_row, start_col,
+                             { end_line = end_row, end_col = end_col,
+                               hl_group = hl,
+                               ephemeral = true
+                              })
     end
     if start_row > line then
       self.nextrow = start_row
@@ -166,7 +172,7 @@ function TSHighlighter._on_line(_, _win, buf, line)
   end
 end
 
-function TSHighlighter._on_start(_, buf, _tick)
+function TSHighlighter._on_buf(_, buf)
   local self = TSHighlighter.active[buf]
   if self then
     local tree = self.parser:parse()
@@ -187,10 +193,10 @@ function TSHighlighter._on_win(_, _win, buf, _topline, botline)
   return true
 end
 
-a.nvim__set_luahl {
+a.nvim_set_decoration_provider(ns, {
-  on_start = TSHighlighter._on_start;
+  on_buf = TSHighlighter._on_buf;
   on_win = TSHighlighter._on_win;
   on_line = TSHighlighter._on_line;
-}
+})
 
 return TSHighlighter

src/nvim/api/buffer.c

@@ -7,6 +7,7 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <limits.h>
+
 #include <lauxlib.h>
 
 #include "nvim/api/buffer.h"
@@ -1285,6 +1286,10 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id,
 ///               - hl_group : name of the highlight group used to highlight
 ///                   this mark.
 ///               - virt_text : virtual text to link to this mark.
+///               - ephemeral : for use with |nvim_set_decoration_provider|
+///                   callbacks. The mark will only be used for the current
+///                   redraw cycle, and not be permantently stored in the
+///                   buffer.
 /// @param[out]  err   Error details, if any
 /// @return Id of the created/updated extmark
 Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
@@ -1318,6 +1323,8 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
     return 0;
   }
 
+  bool ephemeral = false;
+
   uint64_t id = 0;
   int line2 = -1, hl_id = 0;
   colnr_T col2 = 0;
@@ -1386,6 +1393,11 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
       if (ERROR_SET(err)) {
         goto error;
       }
+    } else if (strequal("ephemeral", k.data)) {
+      ephemeral = api_is_truthy(*v, "ephemeral", false, err);
+      if (ERROR_SET(err)) {
+        goto error;
+      }
     } else {
       api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
       goto error;
@@ -1410,17 +1422,33 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
     col2 = 0;
   }
 
-  Decoration *decor = NULL;
+  // TODO(bfredl): synergize these two branches even more
-  if (kv_size(virt_text)) {
+  if (ephemeral && redrawn_win && redrawn_win->w_buffer == buf) {
-    decor = xcalloc(1, sizeof(*decor));
+    int attr_id = hl_id > 0 ? syn_id2attr(hl_id) : 0;
-    decor->hl_id = hl_id;
+    VirtText *vt_allocated = NULL;
-    decor->virt_text = virt_text;
+    if (kv_size(virt_text)) {
-  } else if (hl_id) {
+      vt_allocated = xmalloc(sizeof *vt_allocated);
-    decor = decoration_hl(hl_id);
+      *vt_allocated = virt_text;
-  }
+    }
+    decorations_add_ephemeral(attr_id, (int)line, (colnr_T)col,
+                              (int)line2, (colnr_T)col2, vt_allocated);
+  } else {
+    if (ephemeral) {
+      api_set_error(err, kErrorTypeException, "not yet implemented");
+      goto error;
+    }
+    Decoration *decor = NULL;
+    if (kv_size(virt_text)) {
+      decor = xcalloc(1, sizeof(*decor));
+      decor->hl_id = hl_id;
+      decor->virt_text = virt_text;
+    } else if (hl_id) {
+      decor = decoration_hl(hl_id);
+    }
 
-  id = extmark_set(buf, (uint64_t)ns_id, id,
+    id = extmark_set(buf, (uint64_t)ns_id, id, (int)line, (colnr_T)col,
-                   (int)line, (colnr_T)col, line2, col2, decor, kExtmarkNoUndo);
+                     line2, col2, decor, kExtmarkNoUndo);
+  }
 
   return (Integer)id;
 

src/nvim/api/private/helpers.c

@@ -1619,14 +1619,28 @@ free_exit:
   return virt_text;
 }
 
-bool api_is_truthy(Object obj, const char *what, Error *err)
+bool api_is_truthy(Object obj, const char *what, bool nil_truthy, Error *err)
 {
   if (obj.type == kObjectTypeBoolean) {
     return obj.data.boolean;
   } else if (obj.type == kObjectTypeInteger) {
-    return obj.data.integer;  // C semantics: non-zery int is true
+    return obj.data.integer;  // C semantics: non-zero int is true
+  } else if (obj.type == kObjectTypeNil) {
+    return nil_truthy;  // caller decides what NIL (missing retval in lua) means
   } else {
     api_set_error(err, kErrorTypeValidation, "%s is not an boolean", what);
     return false;
   }
 }
+
+const char *describe_ns(NS ns_id)
+{
+  String name;
+  handle_T id;
+  map_foreach(namespace_ids, name, id, {
+    if ((NS)id == ns_id && name.size) {
+      return name.data;
+    }
+  })
+  return "(UNKNOWN PLUGIN)";
+}

src/nvim/api/vim.c

@@ -2604,166 +2604,134 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Error *err)
   return ret;
 }
 
-/// Set attrs in nvim__buf_set_lua_hl callbacks
-///
-/// TODO(bfredl): This is rather pedestrian. The final
-/// interface should probably be derived from a reformed
-/// bufhl/virttext interface with full support for multi-line
-/// ranges etc
-void nvim__put_attr(Integer line, Integer col, Dictionary opts, Error *err)
-  FUNC_API_LUA_ONLY
-{
-  if (!lua_attr_active) {
-    return;
-  }
-  int line2 = -1, hl_id = 0;
-  colnr_T col2 = 0;
-  VirtText virt_text = KV_INITIAL_VALUE;
-  for (size_t i = 0; i < opts.size; i++) {
-    String k = opts.items[i].key;
-    Object *v = &opts.items[i].value;
-    if (strequal("end_line", k.data)) {
-      if (v->type != kObjectTypeInteger) {
-        api_set_error(err, kErrorTypeValidation,
-                      "end_line is not an integer");
-        goto error;
-      }
-      if (v->data.integer < 0) {
-        api_set_error(err, kErrorTypeValidation,
-                      "end_line value outside range");
-        goto error;
-      }
-
-      line2 = (int)v->data.integer;
-    } else if (strequal("end_col", k.data)) {
-      if (v->type != kObjectTypeInteger) {
-        api_set_error(err, kErrorTypeValidation,
-                      "end_col is not an integer");
-        goto error;
-      }
-      if (v->data.integer < 0 || v->data.integer > MAXCOL) {
-        api_set_error(err, kErrorTypeValidation,
-                      "end_col value outside range");
-        goto error;
-      }
-
-      col2 = (colnr_T)v->data.integer;
-    } else if (strequal("hl_group", k.data)) {
-      String hl_group;
-      switch (v->type) {
-        case kObjectTypeString:
-          hl_group = v->data.string;
-          hl_id = syn_check_group(
-              (char_u *)(hl_group.data),
-              (int)hl_group.size);
-          break;
-        case kObjectTypeInteger:
-          hl_id = (int)v->data.integer;
-          break;
-        default:
-          api_set_error(err, kErrorTypeValidation,
-                        "hl_group is not valid.");
-          goto error;
-      }
-    } else if (strequal("virt_text", k.data)) {
-      if (v->type != kObjectTypeArray) {
-        api_set_error(err, kErrorTypeValidation,
-                      "virt_text is not an Array");
-        goto error;
-      }
-      virt_text = parse_virt_text(v->data.array, err);
-      if (ERROR_SET(err)) {
-        goto error;
-      }
-    } else {
-      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
-      goto error;
-    }
-  }
-  if (col2 && line2 < 0) {
-    line2 = (int)line;
-  }
-
-  int attr = hl_id ? syn_id2attr((int)hl_id) : 0;
-  if (attr == 0 && !kv_size(virt_text)) {
-    return;
-  }
-
-  VirtText *v = xmalloc(sizeof(*v));
-  *v = virt_text;  // LeakSanitizer be sad
-  decorations_add_luahl_attr(attr, (int)line, (colnr_T)col,
-                             (int)line2, (colnr_T)col2, v);
-error:
-  return;
-}
-
 void nvim__screenshot(String path)
   FUNC_API_FAST
 {
   ui_call_screenshot(path);
 }
 
-static void clear_luahl(bool force)
+static DecorationProvider *get_provider(NS ns_id, bool force)
 {
-  if (luahl_active || force) {
+  ssize_t i;
-    api_free_luaref(luahl_start);
+  for (i = 0; i < (ssize_t)kv_size(decoration_providers); i++) {
-    api_free_luaref(luahl_win);
+    DecorationProvider *item = &kv_A(decoration_providers, i);
-    api_free_luaref(luahl_line);
+    if (item->ns_id == ns_id) {
-    api_free_luaref(luahl_end);
+      return item;
+    } else if (item->ns_id > ns_id) {
+      break;
+    }
   }
-  luahl_start = LUA_NOREF;
+
-  luahl_win = LUA_NOREF;
+  if (!force) {
-  luahl_line = LUA_NOREF;
+    return NULL;
-  luahl_end = LUA_NOREF;
+  }
-  luahl_active = false;
+
+  for (ssize_t j = (ssize_t)kv_size(decoration_providers)-1; j >= i; j++) {
+    // allocates if needed:
+    kv_a(decoration_providers, (size_t)j+1) = kv_A(decoration_providers, j);
+  }
+  DecorationProvider *item = &kv_a(decoration_providers, (size_t)i);
+  *item = DECORATION_PROVIDER_INIT(ns_id);
+
+  return item;
 }
 
-/// Unstabilized interface for defining syntax hl in lua.
+static void clear_provider(DecorationProvider *p)
-///
-/// This is not yet safe for general use, lua callbacks will need to
-/// be restricted, like textlock and probably other stuff.
-///
-/// The API on_line/nvim__put_attr is quite raw and not intended to be the
-/// final shape. Ideally this should operate on chunks larger than a single
-/// line to reduce interpreter overhead, and generate annotation objects
-/// (bufhl/virttext) on the fly but using the same representation.
-void nvim__set_luahl(DictionaryOf(LuaRef) opts, Error *err)
-  FUNC_API_LUA_ONLY
 {
-  redraw_later(NOT_VALID);
+  NLUA_CLEAR_REF(p->redraw_start);
-  clear_luahl(false);
+  NLUA_CLEAR_REF(p->redraw_buf);
+  NLUA_CLEAR_REF(p->redraw_win);
+  NLUA_CLEAR_REF(p->redraw_line);
+  NLUA_CLEAR_REF(p->redraw_end);
+  p->active = false;
+}
+
+/// Set or change decoration provider for a namespace
+///
+/// This is a very general purpose interface for having lua callbacks
+/// being triggered during the redraw code.
+///
+/// The expected usage is to set extmarks for the currently
+/// redrawn buffer. |nvim_buf_set_extmark| can be called to add marks
+/// on a per-window or per-lines basis. Use the `ephemeral` key to only
+/// use the mark for the current screen redraw (the callback will be called
+/// again for the next redraw ).
+///
+/// Note: this function should not be called often. Rather, the callbacks
+/// themselves can be used to throttle unneeded callbacks. the `on_start`
+/// callback can return `false` to disable the provider until the next redraw.
+/// Similarily, return `false` in `on_win` will skip the `on_lines` calls
+/// for that window (but any extmarks set in `on_win` will still be used).
+/// A plugin managing multiple sources of decorations should ideally only set
+/// one provider, and merge the sources internally. You can use multiple `ns_id`
+/// for the extmarks set/modified inside the callback anyway.
+///
+/// Note: doing anything other than setting extmarks is considered experimental.
+/// Doing things like changing options are not expliticly forbidden, but is
+/// likely to have unexpected consequences (such as 100% CPU consumption).
+/// doing `vim.rpcnotify` should be OK, but `vim.rpcrequest` is quite dubious
+/// for the moment.
+///
+/// @param ns_id  Namespace id from |nvim_create_namespace()|
+/// @param opts   Callbacks invoked during redraw:
+///             - on_start: called first on each screen redraw
+///                 ["start", tick]
+///             - on_buf: called for each buffer being redrawn (before window
+///                 callbacks)
+///                 ["buf", bufnr, tick]
+///             - on_win: called when starting to redraw a specific window.
+///                 ["win", winid, bufnr, topline, botline_guess]
+///             - on_line: called for each buffer line being redrawn. (The
+///                 interation with fold lines is subject to change)
+///                 ["win", winid, bufnr, row]
+///             - on_end: called at the end of a redraw cycle
+///                 ["end", tick]
+void nvim_set_decoration_provider(Integer ns_id, DictionaryOf(LuaRef) opts,
+                                  Error *err)
+  FUNC_API_SINCE(7) FUNC_API_LUA_ONLY
+{
+  DecorationProvider *p = get_provider((NS)ns_id, true);
+  clear_provider(p);
+
+  // regardless of what happens, it seems good idea to redraw
+  redraw_later(NOT_VALID);  // TODO(bfredl): too soon?
+
+  struct {
+    const char *name;
+    LuaRef *dest;
+  } cbs[] = {
+    { "on_start", &p->redraw_start },
+    { "on_buf", &p->redraw_buf },
+    { "on_win", &p->redraw_win },
+    { "on_line", &p->redraw_line },
+    { "on_end", &p->redraw_end },
+    { NULL, NULL },
+  };
 
   for (size_t i = 0; i < opts.size; i++) {
     String k = opts.items[i].key;
     Object *v = &opts.items[i].value;
-    if (strequal("on_start", k.data)) {
+    size_t j;
-      if (v->type != kObjectTypeLuaRef) {
+    for (j = 0; cbs[j].name; j++) {
-        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+      if (strequal(cbs[j].name, k.data)) {
-        goto error;
+        if (v->type != kObjectTypeLuaRef) {
+          api_set_error(err, kErrorTypeValidation,
+                        "%s is not a function", cbs[j].name);
+          goto error;
+        }
+        *(cbs[j].dest) = v->data.luaref;
+        v->data.luaref = LUA_NOREF;
+        break;
       }
-      luahl_start = v->data.luaref;
+    }
-      v->data.luaref = LUA_NOREF;
+    if (!cbs[j].name) {
-    } else if (strequal("on_win", k.data)) {
-      if (v->type != kObjectTypeLuaRef) {
-        api_set_error(err, kErrorTypeValidation, "callback is not a function");
-        goto error;
-      }
-      luahl_win = v->data.luaref;
-      v->data.luaref = LUA_NOREF;
-    } else if (strequal("on_line", k.data)) {
-      if (v->type != kObjectTypeLuaRef) {
-        api_set_error(err, kErrorTypeValidation, "callback is not a function");
-        goto error;
-      }
-      luahl_line = v->data.luaref;
-      v->data.luaref = LUA_NOREF;
-    } else {
       api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
       goto error;
     }
   }
-  luahl_active = true;
+
+  p->active = true;
   return;
 error:
-  clear_luahl(true);
+  clear_provider(p);
 }

src/nvim/buffer_defs.h

@@ -544,6 +544,9 @@ struct file_buffer {
   long b_mod_xlines;            // number of extra buffer lines inserted;
                                 // negative when lines were deleted
   wininfo_T   *b_wininfo;       // list of last used info for each window
+  int b_mod_tick_syn;           // last display tick syntax was updated
+  int b_mod_tick_deco;          // last display tick decoration providers
+                                // where invoked
 
   long b_mtime;                 // last change time of original file
   long b_mtime_read;            // last change time when reading

src/nvim/extmark.h

@@ -1,6 +1,7 @@
 #ifndef NVIM_EXTMARK_H
 #define NVIM_EXTMARK_H
 
+#include "nvim/pos.h"
 #include "nvim/buffer_defs.h"
 #include "nvim/extmark_defs.h"
 #include "nvim/marktree.h"
@@ -98,6 +99,8 @@ typedef struct {
   VirtText *virt_text;
 } DecorationRedrawState;
 
+EXTERN kvec_t(DecorationProvider) decoration_providers INIT(= KV_INITIAL_VALUE);
+EXTERN win_T *redrawn_win INIT(= NULL);  // used for ephemeral extmarks
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "extmark.h.generated.h"

src/nvim/extmark_defs.h

@@ -1,7 +1,7 @@
 #ifndef NVIM_EXTMARK_DEFS_H
 #define NVIM_EXTMARK_DEFS_H
 
-#include "nvim/pos.h"  // for colnr_T
+#include "nvim/types.h"
 #include "nvim/lib/kvec.h"
 
 typedef struct {
@@ -42,4 +42,18 @@ typedef enum {
   kExtmarkUndoNoRedo,  // Operation should be undoable, but not redoable
 } ExtmarkOp;
 
+typedef struct {
+  NS ns_id;
+  bool active;
+  LuaRef redraw_start;
+  LuaRef redraw_buf;
+  LuaRef redraw_win;
+  LuaRef redraw_line;
+  LuaRef redraw_end;
+} DecorationProvider;
+
+#define DECORATION_PROVIDER_INIT(ns_id) (DecorationProvider) \
+                                 { ns_id, false, LUA_NOREF, LUA_NOREF, \
+                                   LUA_NOREF, LUA_NOREF, LUA_NOREF }
+
 #endif  // NVIM_EXTMARK_DEFS_H

src/nvim/globals.h

@@ -125,8 +125,6 @@ typedef off_t off_T;
 EXTERN int mod_mask INIT(= 0x0);  // current key modifiers
 
 
-EXTERN bool lua_attr_active INIT(= false);
-
 // Cmdline_row is the row where the command line starts, just below the
 // last window.
 // When the cmdline gets longer than the available space the screen gets
@@ -405,12 +403,6 @@ EXTERN int sys_menu INIT(= false);
 // ('lines' and 'rows') must not be changed.
 EXTERN int updating_screen INIT(= 0);
 
-EXTERN bool luahl_active INIT(= false);
-EXTERN LuaRef luahl_start INIT(= LUA_NOREF);
-EXTERN LuaRef luahl_win INIT(= LUA_NOREF);
-EXTERN LuaRef luahl_line INIT(= LUA_NOREF);
-EXTERN LuaRef luahl_end INIT(= LUA_NOREF);
-
 // All windows are linked in a list. firstwin points to the first entry,
 // lastwin to the last entry (can be the same as firstwin) and curwin to the
 // currently active window.

src/nvim/lua/executor.h

@@ -24,6 +24,15 @@ EXTERN LuaRef nlua_empty_dict_ref INIT(= LUA_NOREF);
       memcpy(&err_->msg[0], s, sizeof(s)); \
     } while (0)
 
+#define NLUA_CLEAR_REF(x) \
+  do { \
+    /* Take the address to avoid double evaluation. #1375 */ \
+    if ((x) != LUA_NOREF) { \
+      api_free_luaref(x); \
+      (x) = LUA_NOREF; \
+    } \
+  } while (0)
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "lua/executor.h.generated.h"
 #endif

src/nvim/map.c

@@ -183,7 +183,6 @@ MAP_IMPL(uint64_t, ssize_t, SSIZE_INITIALIZER)
 MAP_IMPL(uint64_t, uint64_t, DEFAULT_INITIALIZER)
 #define EXTMARK_NS_INITIALIZER { 0, 0 }
 MAP_IMPL(uint64_t, ExtmarkNs, EXTMARK_NS_INITIALIZER)
-#define KVEC_INITIALIZER { .size = 0, .capacity = 0, .items = NULL }
 #define EXTMARK_ITEM_INITIALIZER { 0, 0, NULL }
 MAP_IMPL(uint64_t, ExtmarkItem, EXTMARK_ITEM_INITIALIZER)
 MAP_IMPL(handle_T, ptr_t, DEFAULT_INITIALIZER)

src/nvim/marktree.h

@@ -2,6 +2,7 @@
 #define NVIM_MARKTREE_H
 
 #include <stdint.h>
+#include "nvim/pos.h"
 #include "nvim/map.h"
 #include "nvim/garray.h"
 

src/nvim/screen.c

@@ -119,10 +119,12 @@
 #include "nvim/api/private/helpers.h"
 #include "nvim/api/vim.h"
 #include "nvim/lua/executor.h"
+#include "nvim/lib/kvec.h"
 
 #define MB_FILLER_CHAR '<'  /* character used when a double-width character
                              * doesn't fit. */
 
+typedef kvec_withinit_t(DecorationProvider *, 4) Providers;
 
 // temporary buffer for rendering a single screenline, so it can be
 // compared with previous contents to calculate smallest delta.
@@ -156,13 +158,43 @@ static bool msg_grid_invalid = false;
 
 static bool resizing = false;
 
-static bool do_luahl_line = false;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "screen.c.generated.h"
 #endif
 #define SEARCH_HL_PRIORITY 0
 
+static char * provider_first_error = NULL;
+
+static bool provider_invoke(NS ns_id, const char *name, LuaRef ref,
+                            Array args, bool default_true)
+{
+  Error err = ERROR_INIT;
+
+  textlock++;
+  Object ret = nlua_call_ref(ref, name, args, true, &err);
+  textlock--;
+
+  if (!ERROR_SET(&err)
+      && api_is_truthy(ret, "provider %s retval", default_true, &err)) {
+    return true;
+  }
+
+  if (ERROR_SET(&err)) {
+    const char *ns_name = describe_ns(ns_id);
+    ELOG("error in provider %s:%s: %s", ns_name, name, err.msg);
+    bool verbose_errs = true;  // TODO(bfredl):
+    if (verbose_errs && provider_first_error == NULL) {
+      static char errbuf[IOSIZE];
+      snprintf(errbuf, sizeof errbuf, "%s: %s", ns_name, err.msg);
+      provider_first_error = xstrdup(errbuf);
+    }
+  }
+
+  api_free_object(ret);
+  return false;
+}
+
 /*
  * Redraw the current window later, with update_screen(type).
  * Set must_redraw only if not already set to a higher value.
@@ -446,6 +478,29 @@ int update_screen(int type)
 
   ui_comp_set_screen_valid(true);
 
+  Providers providers;
+  kvi_init(providers);
+  for (size_t i = 0; i < kv_size(decoration_providers); i++) {
+    DecorationProvider *p = &kv_A(decoration_providers, i);
+    if (!p->active) {
+      continue;
+    }
+
+    bool active;
+    if (p->redraw_start != LUA_NOREF) {
+      FIXED_TEMP_ARRAY(args, 2);
+      args.items[0] = INTEGER_OBJ(display_tick);
+      args.items[1] = INTEGER_OBJ(type);
+      active = provider_invoke(p->ns_id, "start", p->redraw_start, args, true);
+    } else {
+      active = true;
+    }
+
+    if (active) {
+      kvi_push(providers, p);
+    }
+  }
+
   if (clear_cmdline)            /* going to clear cmdline (done below) */
     check_for_delay(FALSE);
 
@@ -494,30 +549,24 @@ int update_screen(int type)
   FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
     update_window_hl(wp, type >= NOT_VALID);
 
-    if (wp->w_buffer->b_mod_set) {
+    buf_T *buf = wp->w_buffer;
-      win_T       *wwp;
+    if (buf->b_mod_set) {
-
+      if (buf->b_mod_tick_syn < display_tick
-      // Check if we already did this buffer.
+          && syntax_present(wp)) {
-      for (wwp = firstwin; wwp != wp; wwp = wwp->w_next) {
+        syn_stack_apply_changes(buf);
-        if (wwp->w_buffer == wp->w_buffer) {
+        buf->b_mod_tick_syn = display_tick;
-          break;
-        }
-      }
-      if (wwp == wp && syntax_present(wp)) {
-        syn_stack_apply_changes(wp->w_buffer);
       }
 
-      buf_T *buf = wp->w_buffer;
+      if (buf->b_mod_tick_deco < display_tick) {
-      if (luahl_active && luahl_start != LUA_NOREF) {
+        for (size_t i = 0; i < kv_size(providers); i++) {
-        Error err = ERROR_INIT;
+          DecorationProvider *p = kv_A(providers, i);
-        FIXED_TEMP_ARRAY(args, 2);
+          if (p && p->redraw_buf != LUA_NOREF) {
-        args.items[0] = BUFFER_OBJ(buf->handle);
+            FIXED_TEMP_ARRAY(args, 1);
-        args.items[1] = INTEGER_OBJ(display_tick);
+            args.items[0] = BUFFER_OBJ(buf->handle);
-        nlua_call_ref(luahl_start, "start", args, false, &err);
+            provider_invoke(p->ns_id, "buf", p->redraw_buf, args, true);
-        if (ERROR_SET(&err)) {
+          }
-          ELOG("error in luahl start: %s", err.msg);
-          api_clear_error(&err);
         }
+        buf->b_mod_tick_deco = display_tick;
       }
     }
   }
@@ -531,6 +580,8 @@ int update_screen(int type)
 
 
   FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
+    redrawn_win = wp;
+
     if (wp->w_redr_type == CLEAR && wp->w_floating && wp->w_grid.chars) {
       grid_invalidate(&wp->w_grid);
       wp->w_redr_type = NOT_VALID;
@@ -541,13 +592,15 @@ int update_screen(int type)
         did_one = TRUE;
         start_search_hl();
       }
-      win_update(wp);
+      win_update(wp, &providers);
     }
 
     /* redraw status line after the window to minimize cursor movement */
     if (wp->w_redr_status) {
       win_redr_status(wp);
     }
+
+    redrawn_win = NULL;
   }
 
   end_search_hl();
@@ -578,6 +631,24 @@ int update_screen(int type)
     maybe_intro_message();
   did_intro = TRUE;
 
+  for (size_t i = 0; i < kv_size(providers); i++) {
+    DecorationProvider *p = kv_A(providers, i);
+    if (!p->active) {
+      continue;
+    }
+
+    bool active;
+    if (p->redraw_end != LUA_NOREF) {
+      FIXED_TEMP_ARRAY(args, 1);
+      args.items[0] = INTEGER_OBJ(display_tick);
+      active = provider_invoke(p->ns_id, "end", p->redraw_end, args, true);
+    } else {
+      active = true;
+    }
+  }
+  kvi_destroy(providers);
+
+
   // either cmdline is cleared, not drawn or mode is last drawn
   cmdline_was_last_drawn = false;
   return OK;
@@ -637,13 +708,14 @@ bool win_cursorline_standout(const win_T *wp)
 static DecorationRedrawState decorations;
 bool decorations_active = false;
 
-void decorations_add_luahl_attr(int attr_id,
+void decorations_add_ephemeral(int attr_id,
-                                int start_row, int start_col,
+                               int start_row, int start_col,
-                                int end_row, int end_col, VirtText *virt_text)
+                               int end_row, int end_col, VirtText *virt_text)
 {
   kv_push(decorations.active,
           ((HlRange){ start_row, start_col,
-                      end_row, end_col, attr_id, virt_text, true }));
+                      end_row, end_col,
+                      attr_id, virt_text, virt_text != NULL }));
 }
 
 /*
@@ -673,7 +745,7 @@ void decorations_add_luahl_attr(int attr_id,
  * mid: from mid_start to mid_end (update inversion or changed text)
  * bot: from bot_start to last row (when scrolled up)
  */
-static void win_update(win_T *wp)
+static void win_update(win_T *wp, Providers *providers)
 {
   buf_T       *buf = wp->w_buffer;
   int type;
@@ -1241,31 +1313,26 @@ static void win_update(win_T *wp)
 
   decorations_active = decorations_redraw_reset(buf, &decorations);
 
-  do_luahl_line = false;
+  Providers line_providers;
+  kvi_init(line_providers);
 
-  if (luahl_win != LUA_NOREF) {
+  linenr_T knownmax = ((wp->w_valid & VALID_BOTLINE)
-    Error err = ERROR_INIT;
+                       ? wp->w_botline
-    FIXED_TEMP_ARRAY(args, 4);
+                       : (wp->w_topline + wp->w_height_inner));
-    linenr_T knownmax = ((wp->w_valid & VALID_BOTLINE)
-                         ? wp->w_botline
-                         : (wp->w_topline + wp->w_height_inner));
-    args.items[0] = WINDOW_OBJ(wp->handle);
-    args.items[1] = BUFFER_OBJ(buf->handle);
-    // TODO(bfredl): we are not using this, but should be first drawn line?
-    args.items[2] = INTEGER_OBJ(wp->w_topline-1);
-    args.items[3] = INTEGER_OBJ(knownmax);
-    // TODO(bfredl): we could allow this callback to change mod_top, mod_bot.
-    // For now the "start" callback is expected to use nvim__buf_redraw_range.
-    Object ret = nlua_call_ref(luahl_win, "win", args, true, &err);
 
-    if (!ERROR_SET(&err) && api_is_truthy(ret, "luahl_window retval", &err)) {
+  for (size_t k = 0; k < kv_size(*providers); k++) {
-      do_luahl_line = true;
+    DecorationProvider *p = kv_A(*providers, k);
-      decorations_active = true;
+    if (p && p->redraw_win != LUA_NOREF) {
-    }
+      FIXED_TEMP_ARRAY(args, 4);
-
+      args.items[0] = WINDOW_OBJ(wp->handle);
-    if (ERROR_SET(&err)) {
+      args.items[1] = BUFFER_OBJ(buf->handle);
-      ELOG("error in luahl window: %s", err.msg);
+      // TODO(bfredl): we are not using this, but should be first drawn line?
-      api_clear_error(&err);
+      args.items[2] = INTEGER_OBJ(wp->w_topline-1);
+      args.items[3] = INTEGER_OBJ(knownmax);
+      if (provider_invoke(p->ns_id, "win", p->redraw_win, args, true)) {
+        kvi_push(line_providers, p);
+        decorations_active = true;
+      }
     }
   }
 
@@ -1483,7 +1550,7 @@ static void win_update(win_T *wp)
         // Display one line
         row = win_line(wp, lnum, srow,
                        foldinfo.fi_lines ? srow : wp->w_grid.Rows,
-                       mod_top == 0, false, foldinfo);
+                       mod_top == 0, false, foldinfo, &line_providers);
 
         wp->w_lines[idx].wl_folded = foldinfo.fi_lines != 0;
         wp->w_lines[idx].wl_lastlnum = lnum;
@@ -1519,7 +1586,8 @@ static void win_update(win_T *wp)
         // 'relativenumber' set: The text doesn't need to be drawn, but
         // the number column nearly always does.
         foldinfo_T info = fold_info(wp, lnum);
-        (void)win_line(wp, lnum, srow, wp->w_grid.Rows, true, true, info);
+        (void)win_line(wp, lnum, srow, wp->w_grid.Rows, true, true,
+                       info, &line_providers);
       }
 
       // This line does not need to be drawn, advance to the next one.
@@ -1615,6 +1683,8 @@ static void win_update(win_T *wp)
                  HLF_EOB);
   }
 
+  kvi_destroy(line_providers);
+
   if (wp->w_redr_type >= REDRAW_TOP) {
     draw_vsep_win(wp, 0);
   }
@@ -1674,7 +1744,7 @@ static void win_update(win_T *wp)
         curbuf->b_mod_set = false;
         j = curbuf->b_mod_xlines;
         curbuf->b_mod_xlines = 0;
-        win_update(curwin);
+        win_update(curwin, providers);
         curbuf->b_mod_set = i;
         curbuf->b_mod_xlines = j;
       }
@@ -1919,28 +1989,23 @@ fill_foldcolumn(
   return MAX(char_counter + (fdc-i), (size_t)fdc);
 }
 
-
 /// Display line "lnum" of window 'wp' on the screen.
-/// Start at row "startrow", stop when "endrow" is reached.
 /// wp->w_virtcol needs to be valid.
 ///
-/// @param lnum line to display
+/// @param lnum         line to display
-/// @param endrow stop drawing once reaching this row
+/// @param startrow     first row relative to window grid
-/// @param nochange not updating for changed text
+/// @param endrow       last grid row to be redrawn
-/// @param number_only only update the number column
+/// @param nochange     not updating for changed text
-/// @param foldinfo fold info for this line
+/// @param number_only  only update the number column
+/// @param foldinfo     fold info for this line
+/// @param[in, out] providers  decoration providers active this line
+///                            items will be disables if they cause errors
+///                            or explicitly return `false`.
 ///
-/// @return the number of last row the line occupies.
+/// @return             the number of last row the line occupies.
-static int
+static int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow,
-win_line (
+                    bool nochange, bool number_only, foldinfo_T foldinfo,
-    win_T *wp,
+                    Providers *providers)
-    linenr_T lnum,
-    int startrow,
-    int endrow,
-    bool nochange,
-    bool number_only,
-    foldinfo_T foldinfo
-)
 {
   int c = 0;                          // init for GCC
   long vcol = 0;                      // virtual column (for tabs)
@@ -2084,7 +2149,7 @@ win_line (
 
   row = startrow;
 
-  char *luatext = NULL;
+  char *err_text = NULL;
 
   buf_T *buf = wp->w_buffer;
 
@@ -2109,22 +2174,18 @@ win_line (
       }
     }
 
-    if (decorations_active) {
+    for (size_t k = 0; k < kv_size(*providers); k++) {
-      if (do_luahl_line && luahl_line != LUA_NOREF) {
+      DecorationProvider *p = kv_A(*providers, k);
-        Error err = ERROR_INIT;
+      if (p && p->redraw_line != LUA_NOREF) {
         FIXED_TEMP_ARRAY(args, 3);
         args.items[0] = WINDOW_OBJ(wp->handle);
         args.items[1] = BUFFER_OBJ(buf->handle);
         args.items[2] = INTEGER_OBJ(lnum-1);
-        lua_attr_active = true;
+        if (provider_invoke(p->ns_id, "line", p->redraw_line, args, true)) {
-        extra_check = true;
+          decorations_active = true;
-        nlua_call_ref(luahl_line, "line", args, false, &err);
+        } else {
-        lua_attr_active = false;
+          // return 'false' or error: skip rest of this window
-
+          kv_A(*providers, k) = NULL;
-        if (ERROR_SET(&err)) {
-          ELOG("error in luahl line: %s", err.msg);
-          luatext = err.msg;
-          do_virttext = true;
         }
       }
 
@@ -2135,6 +2196,12 @@ win_line (
       }
     }
 
+    if (provider_first_error) {
+      err_text = provider_first_error;
+      provider_first_error = NULL;
+      do_virttext = true;
+    }
+
     // Check for columns to display for 'colorcolumn'.
     color_cols = wp->w_buffer->terminal ? NULL : wp->w_p_cc_cols;
     if (color_cols != NULL) {
@@ -3835,8 +3902,10 @@ win_line (
         draw_color_col = advance_color_col(VCOL_HLC, &color_cols);
 
       VirtText virt_text = KV_INITIAL_VALUE;
-      if (luatext) {
+      if (err_text) {
-        kv_push(virt_text, ((VirtTextChunk){ .text = luatext, .hl_id = 0 }));
+        int hl_err = syn_check_group((char_u *)S_LEN("ErrorMsg"));
+        kv_push(virt_text, ((VirtTextChunk){ .text = err_text,
+                                             .hl_id = hl_err }));
         do_virttext = true;
       } else if (has_decorations) {
         VirtText *vp = decorations_redraw_virt_text(wp->w_buffer, &decorations);
@@ -4256,7 +4325,7 @@ win_line (
   }
 
   xfree(p_extra_free);
-  xfree(luatext);
+  xfree(err_text);
   return row;
 }
 

src/nvim/types.h

@@ -21,6 +21,8 @@ typedef int handle_T;
 // absent callback etc.
 typedef int LuaRef;
 
+typedef uint64_t NS;
+
 typedef struct expand expand_T;
 
 typedef enum {

test/functional/helpers.lua

@@ -22,6 +22,7 @@ local ok = global_helpers.ok
 local sleep = global_helpers.sleep
 local tbl_contains = global_helpers.tbl_contains
 local write_file = global_helpers.write_file
+local fail = global_helpers.fail
 
 local module = {
   NIL = mpack.NIL,
@@ -592,6 +593,24 @@ function module.expect_any(contents)
   return ok(nil ~= string.find(module.curbuf_contents(), contents, 1, true))
 end
 
+function module.expect_events(expected, received, kind)
+  local inspect = require'vim.inspect'
+  if not pcall(eq, expected, received) then
+    local msg = 'unexpected '..kind..' received.\n\n'
+
+    msg = msg .. 'received events:\n'
+    for _, e in ipairs(received) do
+      msg = msg .. '  ' .. inspect(e) .. ';\n'
+    end
+    msg = msg .. '\nexpected events:\n'
+    for _, e in ipairs(expected) do
+      msg = msg .. '  ' .. inspect(e) .. ';\n'
+    end
+    fail(msg)
+  end
+  return received
+end
+
 -- Checks that the Nvim session did not terminate.
 function module.assert_alive()
   assert(2 == module.eval('1+1'), 'crash? request failed')

test/functional/lua/buffer_updates_spec.lua

@@ -1,8 +1,6 @@
 -- Test suite for testing interactions with API bindings
 local helpers = require('test.functional.helpers')(after_each)
 
-local inspect = require'vim.inspect'
-
 local command = helpers.command
 local meths = helpers.meths
 local funcs = helpers.funcs
@@ -12,6 +10,7 @@ local fail = helpers.fail
 local exec_lua = helpers.exec_lua
 local feed = helpers.feed
 local deepcopy = helpers.deepcopy
+local expect_events = helpers.expect_events
 
 local origlines = {"original line 1",
                    "original line 2",
@@ -297,20 +296,7 @@ describe('lua: nvim_buf_attach on_bytes', function()
     local verify_name = "test1"
     local function check_events(expected)
       local events = exec_lua("return get_events(...)" )
-
+      expect_events(expected, events, "byte updates")
-      if not pcall(eq, expected, events) then
-        local msg = 'unexpected byte updates received.\n\n'
-
-        msg = msg .. 'received events:\n'
-        for _, e in ipairs(events) do
-          msg = msg .. '  ' .. inspect(e) .. ';\n'
-        end
-        msg = msg .. '\nexpected events:\n'
-        for _, e in ipairs(expected) do
-          msg = msg .. '  ' .. inspect(e) .. ';\n'
-        end
-        fail(msg)
-      end
 
       if not verify then
         return

test/functional/ui/decorations_spec.lua

@@ -0,0 +1,118 @@
+local helpers = require('test.functional.helpers')(after_each)
+local Screen = require('test.functional.ui.screen')
+
+local clear = helpers.clear
+local feed = helpers.feed
+local insert = helpers.insert
+local exec_lua = helpers.exec_lua
+local expect_events = helpers.expect_events
+
+describe('decorations provider', function()
+  local screen
+  before_each(function()
+    clear()
+    screen = Screen.new(40, 8)
+    screen:attach()
+    screen:set_default_attr_ids({
+      [1] = {bold=true, foreground=Screen.colors.Blue},
+    })
+  end)
+
+  local mudholland = [[
+    // just to see if there was an accident
+    // on Mulholland Drive
+    try_start();
+    bufref_T save_buf;
+    switch_buffer(&save_buf, buf);
+    posp = getmark(mark, false);
+    restore_buffer(&save_buf); ]]
+
+  local function setup_provider(code)
+    exec_lua ([[
+      local a = vim.api
+      test1 = a.nvim_create_namespace "test1"
+    ]] .. (code or [[
+      beamtrace = {}
+      function on_do(kind, ...)
+        table.insert(beamtrace, {kind, ...})
+      end
+    ]]) .. [[
+      a.nvim_set_decoration_provider(
+        test1, {
+        on_start = on_do; on_buf = on_do;
+        on_win = on_do; on_line = on_do;
+        on_end = on_do;
+      })
+    ]])
+  end
+
+  local function check_trace(expected)
+    local actual = exec_lua [[ local b = beamtrace beamtrace = {} return b ]]
+    expect_events(expected, actual, "beam trace")
+  end
+
+  it('leaves a trace', function()
+    insert(mudholland)
+
+    setup_provider()
+
+    screen:expect{grid=[[
+      // just to see if there was an accident |
+      // on Mulholland Drive                  |
+      try_start();                            |
+      bufref_T save_buf;                      |
+      switch_buffer(&save_buf, buf);          |
+      posp = getmark(mark, false);            |
+      restore_buffer(&save_buf);^              |
+                                              |
+    ]]}
+    check_trace {
+      { "start", 4, 40 };
+      { "win", 1000, 1, 0, 8 };
+      { "line", 1000, 1, 0 };
+      { "line", 1000, 1, 1 };
+      { "line", 1000, 1, 2 };
+      { "line", 1000, 1, 3 };
+      { "line", 1000, 1, 4 };
+      { "line", 1000, 1, 5 };
+      { "line", 1000, 1, 6 };
+      { "end", 4 };
+    }
+
+    feed "iü<esc>"
+    screen:expect{grid=[[
+      // just to see if there was an accident |
+      // on Mulholland Drive                  |
+      try_start();                            |
+      bufref_T save_buf;                      |
+      switch_buffer(&save_buf, buf);          |
+      posp = getmark(mark, false);            |
+      restore_buffer(&save_buf);^ü             |
+                                              |
+    ]]}
+    check_trace {
+      { "start", 5, 10 };
+      { "buf", 1 };
+      { "win", 1000, 1, 0, 8 };
+      { "line", 1000, 1, 6 };
+      { "end", 5 };
+    }
+  end)
+
+  it('single provider', function()
+    insert(mudholland)
+    setup_provider [[
+      local hl = a.nvim_get_hl_id_by_name "ErrorMsg"
+      function do_it(event, ...)
+        if event == "line" then
+          local win, buf, line = ...
+          a.nvim_buf_set_extmark(buf, test_ns, line, line,
+                             { end_line = line, end_col = line+1,
+                               hl_group = hl,
+                               ephemeral = true
+                              })
+        end
+      end
+    ]]
+  end)
+end)