Merge pull request #12816 from vigoux/decorations

New Decorations API (not finalized, but we gonna build on this)

runtime/lua/vim/treesitter/highlighter.lua

@@ -98,7 +98,7 @@ function TSHighlighter:get_hl_from_capture(capture)
     return vim.split(name, '.', true)[1]
   else
     -- Default to false to avoid recomputing
-    return TSHighlighter.hl_map[name]
+    return a.nvim_get_hl_id_by_name(TSHighlighter.hl_map[name])
   end
 end
 
@@ -142,10 +142,11 @@ function TSHighlighter:on_changedtree(changes)
       local start_row, start_col, end_row, end_col = node:range()
       local hl = self.hl_cache[capture]
       if hl then
-        a.nvim__buf_add_decoration(self.buf, ts_hs_ns, hl,
+        a.nvim_buf_set_extmark(self.buf, ts_hs_ns, start_row, start_col, {
-          start_row, start_col,
+          end_col = end_col,
-          end_row, end_col,
+          end_line = end_row,
-          {})
+          hl_group = hl
+        })
       end
     end
   end

src/nvim/api/buffer.c

@@ -1108,15 +1108,65 @@ ArrayOf(Integer, 2) nvim_buf_get_mark(Buffer buffer, String name, Error *err)
   return rv;
 }
 
+static Array extmark_to_array(ExtmarkInfo extmark, bool id, bool add_dict)
+{
+  Array rv = ARRAY_DICT_INIT;
+  if (id) {
+    ADD(rv, INTEGER_OBJ((Integer)extmark.mark_id));
+  }
+  ADD(rv, INTEGER_OBJ(extmark.row));
+  ADD(rv, INTEGER_OBJ(extmark.col));
+
+  if (add_dict) {
+    Dictionary dict = ARRAY_DICT_INIT;
+
+    if (extmark.end_row >= 0) {
+      PUT(dict, "end_row", INTEGER_OBJ(extmark.end_row));
+      PUT(dict, "end_col", INTEGER_OBJ(extmark.end_col));
+    }
+
+    if (extmark.decor) {
+      Decoration *decor = extmark.decor;
+      if (decor->hl_id) {
+        String name = cstr_to_string((const char *)syn_id2name(decor->hl_id));
+        PUT(dict, "hl_group", STRING_OBJ(name));
+      }
+      if (kv_size(decor->virt_text)) {
+        Array chunks = ARRAY_DICT_INIT;
+        for (size_t i = 0; i < decor->virt_text.size; i++) {
+          Array chunk = ARRAY_DICT_INIT;
+          VirtTextChunk *vtc = &decor->virt_text.items[i];
+          ADD(chunk, STRING_OBJ(cstr_to_string(vtc->text)));
+          if (vtc->hl_id > 0) {
+            ADD(chunk,
+                STRING_OBJ(cstr_to_string(
+                    (const char *)syn_id2name(vtc->hl_id))));
+          }
+          ADD(chunks, ARRAY_OBJ(chunk));
+        }
+        PUT(dict, "virt_text", ARRAY_OBJ(chunks));
+      }
+    }
+
+    if (dict.size) {
+      ADD(rv, DICTIONARY_OBJ(dict));
+    }
+  }
+
+  return rv;
+}
+
 /// Returns position for a given extmark id
 ///
 /// @param buffer  Buffer handle, or 0 for current buffer
 /// @param ns_id  Namespace id from |nvim_create_namespace()|
 /// @param id  Extmark id
+/// @param details Wether to include the details dict
 /// @param[out] err   Error details, if any
 /// @return (row, col) tuple or empty list () if extmark id was absent
 ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
-                                            Integer id, Error *err)
+                                            Integer id, Boolean details,
+                                            Error *err)
   FUNC_API_SINCE(7)
 {
   Array rv = ARRAY_DICT_INIT;
@@ -1136,9 +1186,7 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
   if (extmark.row < 0) {
     return rv;
   }
-  ADD(rv, INTEGER_OBJ((Integer)extmark.row));
+  return extmark_to_array(extmark, false, (bool)details);
-  ADD(rv, INTEGER_OBJ((Integer)extmark.col));
-  return rv;
 }
 
 /// Gets extmarks in "traversal order" from a |charwise| region defined by
@@ -1181,10 +1229,13 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
 ///             (whose position defines the bound)
 /// @param opts  Optional parameters. Keys:
 ///          - limit:  Maximum number of marks to return
+/// @param details Wether to include the details dict
 /// @param[out] err   Error details, if any
 /// @return List of [extmark_id, row, col] tuples in "traversal order".
-Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
+Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id,
-                            Object end, Dictionary opts, Error *err)
+                            Object start, Object end,
+                            Dictionary opts, Boolean details,
+                            Error *err)
   FUNC_API_SINCE(7)
 {
   Array rv = ARRAY_DICT_INIT;
@@ -1241,16 +1292,11 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
   }
 
 
-  ExtmarkArray marks = extmark_get(buf, (uint64_t)ns_id, l_row, l_col, u_row,
+  ExtmarkInfoArray marks = extmark_get(buf, (uint64_t)ns_id, l_row, l_col,
-                                   u_col, (int64_t)limit, reverse);
+                                       u_row, u_col, (int64_t)limit, reverse);
 
   for (size_t i = 0; i < kv_size(marks); i++) {
-    Array mark = ARRAY_DICT_INIT;
+    ADD(rv, ARRAY_OBJ(extmark_to_array(kv_A(marks, i), true, (bool)details)));
-    ExtmarkInfo extmark = kv_A(marks, i);
-    ADD(mark, INTEGER_OBJ((Integer)extmark.mark_id));
-    ADD(mark, INTEGER_OBJ(extmark.row));
-    ADD(mark, INTEGER_OBJ(extmark.col));
-    ADD(rv, ARRAY_OBJ(mark));
   }
 
   kv_destroy(marks);
@@ -1260,27 +1306,36 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
 /// Creates or updates an extmark.
 ///
 /// To create a new extmark, pass id=0. The extmark id will be returned.
-//  To move an existing mark, pass its id.
+/// To move an existing mark, pass its id.
 ///
 /// It is also allowed to create a new mark by passing in a previously unused
 /// id, but the caller must then keep track of existing and unused ids itself.
 /// (Useful over RPC, to avoid waiting for the return value.)
 ///
+/// Using the optional arguments, it is possible to use this to highlight
+/// a range of text, and also to associate virtual text to the mark.
+///
 /// @param buffer  Buffer handle, or 0 for current buffer
 /// @param ns_id  Namespace id from |nvim_create_namespace()|
-/// @param id  Extmark id, or 0 to create new
 /// @param line  Line number where to place the mark
 /// @param col  Column where to place the mark
-/// @param opts  Optional parameters. Currently not used.
+/// @param opts  Optional parameters.
+///               - id : id of the extmark to edit.
+///               - end_line : ending line of the mark, 0-based inclusive.
+///               - end_col : ending col of the mark, 0-based inclusive.
+///               - hl_group : name of the highlight group used to highlight
+///                   this mark.
+///               - virt_text : virtual text to link to this mark.
 /// @param[out]  err   Error details, if any
 /// @return Id of the created/updated extmark
-Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer id,
+Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
                              Integer line, Integer col,
                              Dictionary opts, Error *err)
   FUNC_API_SINCE(7)
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);
   if (!buf) {
+    api_set_error(err, kErrorTypeValidation, "Invalid buffer id");
     return 0;
   }
 
@@ -1289,11 +1344,6 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer id,
     return 0;
   }
 
-  if (opts.size > 0) {
-    api_set_error(err, kErrorTypeValidation, "opts dict isn't empty");
-    return 0;
-  }
-
   size_t len = 0;
   if (line < 0 || line > buf->b_ml.ml_line_count) {
     api_set_error(err, kErrorTypeValidation, "line value outside range");
@@ -1309,18 +1359,113 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer id,
     return 0;
   }
 
-  uint64_t id_num;
+  uint64_t id = 0;
-  if (id >= 0) {
+  int line2 = -1, hl_id = 0;
-    id_num = (uint64_t)id;
+  colnr_T col2 = 0;
-  } else {
+  VirtText virt_text = KV_INITIAL_VALUE;
-    api_set_error(err, kErrorTypeValidation, "Invalid mark id");
+  for (size_t i = 0; i < opts.size; i++) {
-    return 0;
+    String k = opts.items[i].key;
+    Object *v = &opts.items[i].value;
+    if (strequal("id", k.data)) {
+      if (v->type != kObjectTypeInteger || v->data.integer <= 0) {
+        api_set_error(err, kErrorTypeValidation,
+                      "id is not a positive integer");
+        goto error;
       }
 
-  id_num = extmark_set(buf, (uint64_t)ns_id, id_num,
+      id = (uint64_t)v->data.integer;
-                       (int)line, (colnr_T)col, kExtmarkUndo);
+    } else if (strequal("end_line", k.data)) {
+      if (v->type != kObjectTypeInteger) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_line is not an integer");
+        goto error;
+      }
+      if (v->data.integer < 0 || v->data.integer > buf->b_ml.ml_line_count) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_line value outside range");
+        goto error;
+      }
 
-  return (Integer)id_num;
+      line2 = (int)v->data.integer;
+    } else if (strequal("end_col", k.data)) {
+      if (v->type != kObjectTypeInteger) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_col is not an integer");
+        goto error;
+      }
+      if (v->data.integer < 0 || v->data.integer > MAXCOL) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_col value outside range");
+        goto error;
+      }
+
+      col2 = (colnr_T)v->data.integer;
+    } else if (strequal("hl_group", k.data)) {
+      String hl_group;
+      switch (v->type) {
+        case kObjectTypeString:
+          hl_group = v->data.string;
+          hl_id = syn_check_group(
+              (char_u *)(hl_group.data),
+              (int)hl_group.size);
+          break;
+        case kObjectTypeInteger:
+          hl_id = (int)v->data.integer;
+          break;
+        default:
+          api_set_error(err, kErrorTypeValidation,
+                        "hl_group is not valid.");
+          goto error;
+      }
+    } else if (strequal("virt_text", k.data)) {
+      if (v->type != kObjectTypeArray) {
+        api_set_error(err, kErrorTypeValidation,
+                      "virt_text is not an Array");
+        goto error;
+      }
+      virt_text = parse_virt_text(v->data.array, err);
+      if (ERROR_SET(err)) {
+        goto error;
+      }
+    } else {
+      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
+      goto error;
+    }
+  }
+
+  if (col2 >= 0) {
+    if (line2 >= 0) {
+      len = STRLEN(ml_get_buf(buf, (linenr_T)line2+1, false));
+    } else {
+      // reuse len from before
+      line2 = (int)line;
+    }
+    if (col2 > (Integer)len) {
+      api_set_error(err, kErrorTypeValidation,
+                    "end_col value outside range");
+      goto error;
+    }
+  } else if (line2 >= 0) {
+    col2 = 0;
+  }
+
+  Decoration *decor = NULL;
+  if (kv_size(virt_text)) {
+    decor = xcalloc(1, sizeof(*decor));
+    decor->hl_id = hl_id;
+    decor->virt_text = virt_text;
+  } else if (hl_id) {
+    decor = decoration_hl(hl_id);
+  }
+
+  id = extmark_set(buf, (uint64_t)ns_id, id,
+                   (int)line, (colnr_T)col, line2, col2, decor, kExtmarkUndo);
+
+  return (Integer)id;
+
+error:
+  clear_virttext(&virt_text);
+  return 0;
 }
 
 /// Removes an extmark.
@@ -1412,9 +1557,9 @@ Integer nvim_buf_add_highlight(Buffer buffer,
     return src_id;
   }
 
-  int hlg_id = 0;
+  int hl_id = 0;
   if (hl_group.size > 0) {
-    hlg_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
+    hl_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
   } else {
     return src_id;
   }
@@ -1425,10 +1570,10 @@ Integer nvim_buf_add_highlight(Buffer buffer,
     end_line++;
   }
 
-  extmark_add_decoration(buf, ns_id, hlg_id,
+  ns_id = extmark_set(buf, ns_id, 0,
                       (int)line, (colnr_T)col_start,
                       end_line, (colnr_T)col_end,
-                         VIRTTEXT_EMPTY);
+                      decoration_hl(hl_id), kExtmarkUndo);
   return src_id;
 }
 
@@ -1592,115 +1737,13 @@ Integer nvim_buf_set_virtual_text(Buffer buffer,
     return src_id;
   }
 
-  extmark_add_decoration(buf, ns_id, 0,
+  Decoration *decor = xcalloc(1, sizeof(*decor));
-                         (int)line, 0, -1, -1,
+  decor->virt_text = virt_text;
-                         virt_text);
+
+  extmark_set(buf, ns_id, 0, (int)line, 0, -1, -1, decor, kExtmarkUndo);
   return src_id;
 }
 
-/// Get the virtual text (annotation) for a buffer line.
-///
-/// The virtual text is returned as list of lists, whereas the inner lists have
-/// either one or two elements. The first element is the actual text, the
-/// optional second element is the highlight group.
-///
-/// The format is exactly the same as given to nvim_buf_set_virtual_text().
-///
-/// If there is no virtual text associated with the given line, an empty list
-/// is returned.
-///
-/// @param buffer   Buffer handle, or 0 for current buffer
-/// @param line     Line to get the virtual text from (zero-indexed)
-/// @param[out] err Error details, if any
-/// @return         List of virtual text chunks
-Array nvim_buf_get_virtual_text(Buffer buffer, Integer line, Error *err)
-  FUNC_API_SINCE(7)
-{
-  Array chunks = ARRAY_DICT_INIT;
-
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-  if (!buf) {
-    return chunks;
-  }
-
-  if (line < 0 || line >= MAXLNUM) {
-    api_set_error(err, kErrorTypeValidation, "Line number outside range");
-    return chunks;
-  }
-
-  VirtText *virt_text = extmark_find_virttext(buf, (int)line, 0);
-
-  if (!virt_text) {
-    return chunks;
-  }
-
-  for (size_t i = 0; i < virt_text->size; i++) {
-    Array chunk = ARRAY_DICT_INIT;
-    VirtTextChunk *vtc = &virt_text->items[i];
-    ADD(chunk, STRING_OBJ(cstr_to_string(vtc->text)));
-    if (vtc->hl_id > 0) {
-      ADD(chunk, STRING_OBJ(cstr_to_string(
-          (const char *)syn_id2name(vtc->hl_id))));
-    }
-    ADD(chunks, ARRAY_OBJ(chunk));
-  }
-
-  return chunks;
-}
-
-Integer nvim__buf_add_decoration(Buffer buffer, Integer ns_id, String hl_group,
-                                 Integer start_row, Integer start_col,
-                                 Integer end_row, Integer end_col,
-                                 Array virt_text,
-                                 Error *err)
-{
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-  if (!buf) {
-    return 0;
-  }
-
-  if (!ns_initialized((uint64_t)ns_id)) {
-    api_set_error(err, kErrorTypeValidation, "Invalid ns_id");
-    return 0;
-  }
-
-
-  if (start_row < 0 || start_row >= MAXLNUM || end_row > MAXCOL) {
-    api_set_error(err, kErrorTypeValidation, "Line number outside range");
-    return 0;
-  }
-
-  if (start_col < 0 || start_col > MAXCOL || end_col > MAXCOL) {
-    api_set_error(err, kErrorTypeValidation, "Column value outside range");
-    return 0;
-  }
-  if (end_row < 0 || end_col < 0) {
-    end_row = -1;
-    end_col = -1;
-  }
-
-  if (start_row >= buf->b_ml.ml_line_count
-      || end_row >= buf->b_ml.ml_line_count) {
-    // safety check, we can't add marks outside the range
-    return 0;
-  }
-
-  int hlg_id = 0;
-  if (hl_group.size > 0) {
-    hlg_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
-  }
-
-  VirtText vt = parse_virt_text(virt_text, err);
-  if (ERROR_SET(err)) {
-    return 0;
-  }
-
-  uint64_t mark_id = extmark_add_decoration(buf, (uint64_t)ns_id, hlg_id,
-                                            (int)start_row, (colnr_T)start_col,
-                                            (int)end_row, (colnr_T)end_col, vt);
-  return (Integer)mark_id;
-}
-
 Dictionary nvim__buf_stats(Buffer buffer, Error *err)
 {
   Dictionary rv = ARRAY_DICT_INIT;

src/nvim/extmark.c

@@ -1,29 +1,24 @@
 // This is an open source non-commercial project. Dear PVS-Studio, please check
 // it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
 
-// Implements extended marks for plugins. Each mark exists in a btree of
+// Implements extended marks for plugins. Marks sit in a MarkTree
-// lines containing btrees of columns.
+// datastructure which provides both efficient mark insertations/lookups
+// and adjustment to text changes. See marktree.c for more details.
 //
-// The btree provides efficient range lookups.
 // A map of pointers to the marks is used for fast lookup by mark id.
 //
-// Marks are moved by calls to extmark_splice. Additionally mark_adjust
+// Marks are moved by calls to extmark_splice. Some standard interfaces
-// might adjust extmarks to line inserts/deletes.
+// mark_adjust and inserted_bytes already adjust marks, check if these are
+// being used before adding extmark_splice calls!
 //
 // Undo/Redo of marks is implemented by storing the call arguments to
 // extmark_splice. The list of arguments is applied in extmark_apply_undo.
-// The only case where we have to copy extmarks is for the area being effected
+// We have to copy extmark positions when the extmarks are within a
-// by a delete.
+// deleted/changed region.
 //
 // Marks live in namespaces that allow plugins/users to segregate marks
 // from other users.
 //
-// For possible ideas for efficency improvements see:
-// http://blog.atom.io/2015/06/16/optimizing-an-important-atom-primitive.html
-// TODO(bfredl): These ideas could be used for an enhanced btree, which
-// wouldn't need separate line and column layers.
-// Other implementations exist in gtk and tk toolkits.
-//
 // Deleting marks only happens when explicitly calling extmark_del, deleteing
 // over a range of marks will only move the marks. Deleting on a mark will
 // leave it in same position unless it is on the EOL of a line.
@@ -48,6 +43,13 @@
 # include "extmark.c.generated.h"
 #endif
 
+static PMap(uint64_t) *hl_decors;
+
+void extmark_init(void)
+{
+  hl_decors = pmap_new(uint64_t)();
+}
+
 static ExtmarkNs *buf_ns_ref(buf_T *buf, uint64_t ns_id, bool put) {
   if (!buf->b_extmark_ns) {
     if (!put) {
@@ -71,7 +73,8 @@ static ExtmarkNs *buf_ns_ref(buf_T *buf, uint64_t ns_id, bool put) {
 /// must not be used during iteration!
 /// @returns the mark id
 uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t id,
-                     int row, colnr_T col, ExtmarkOp op)
+                     int row, colnr_T col, int end_row, colnr_T end_col,
+                     Decoration *decor, ExtmarkOp op)
 {
   ExtmarkNs *ns = buf_ns_ref(buf, ns_id, true);
   mtpos_t old_pos;
@@ -82,7 +85,7 @@ uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t id,
   } else {
     uint64_t old_mark = map_get(uint64_t, uint64_t)(ns->map, id);
     if (old_mark) {
-      if (old_mark & MARKTREE_PAIRED_FLAG) {
+      if (old_mark & MARKTREE_PAIRED_FLAG || end_row > -1) {
         extmark_del(buf, ns_id, id);
       } else {
         // TODO(bfredl): we need to do more if "revising" a decoration mark.
@@ -90,7 +93,12 @@ uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t id,
         old_pos = marktree_lookup(buf->b_marktree, old_mark, itr);
         assert(itr->node);
         if (old_pos.row == row && old_pos.col == col) {
-          map_del(uint64_t, ExtmarkItem)(buf->b_extmark_index, old_mark);
+          ExtmarkItem it = map_del(uint64_t, ExtmarkItem)(buf->b_extmark_index,
+                                                          old_mark);
+          if (it.decor) {
+            decoration_redraw(buf, row, row, it.decor);
+            free_decoration(it.decor);
+          }
           mark = marktree_revise(buf->b_marktree, itr);
           goto revised;
         }
@@ -101,11 +109,17 @@ uint64_t extmark_set(buf_T *buf, uint64_t ns_id, uint64_t id,
     }
   }
 
+  if (end_row > -1) {
+    mark = marktree_put_pair(buf->b_marktree,
+                             row, col, true,
+                             end_row, end_col, false);
+  } else {
     mark = marktree_put(buf->b_marktree, row, col, true);
+  }
+
 revised:
   map_put(uint64_t, ExtmarkItem)(buf->b_extmark_index, mark,
-                                 (ExtmarkItem){ ns_id, id, 0,
+                                 (ExtmarkItem){ ns_id, id, decor });
-                                                KV_INITIAL_VALUE });
   map_put(uint64_t, uint64_t)(ns->map, id, mark);
 
   if (op != kExtmarkNoUndo) {
@@ -114,6 +128,10 @@ revised:
     // adding new marks to old undo headers.
     u_extmark_set(buf, mark, row, col);
   }
+
+  if (decor) {
+    decoration_redraw(buf, row, end_row > -1 ? end_row : row, decor);
+  }
   return id;
 }
 
@@ -152,27 +170,23 @@ bool extmark_del(buf_T *buf, uint64_t ns_id, uint64_t id)
   assert(pos.row >= 0);
   marktree_del_itr(buf->b_marktree, itr, false);
   ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index, mark);
+  mtpos_t pos2 = pos;
 
   if (mark & MARKTREE_PAIRED_FLAG) {
-    mtpos_t pos2 = marktree_lookup(buf->b_marktree,
+    pos2 = marktree_lookup(buf->b_marktree, mark|MARKTREE_END_FLAG, itr);
-                                   mark|MARKTREE_END_FLAG, itr);
     assert(pos2.row >= 0);
     marktree_del_itr(buf->b_marktree, itr, false);
-    if (item.hl_id && pos2.row >= pos.row) {
-      redraw_buf_range_later(buf, pos.row+1, pos2.row+1);
-    }
   }
 
-  if (kv_size(item.virt_text)) {
+  if (item.decor) {
-    redraw_buf_line_later(buf, pos.row+1);
+    decoration_redraw(buf, pos.row, pos2.row, item.decor);
+    free_decoration(item.decor);
   }
-  clear_virttext(&item.virt_text);
 
   map_del(uint64_t, uint64_t)(ns->map, id);
   map_del(uint64_t, ExtmarkItem)(buf->b_extmark_index, mark);
 
   // TODO(bfredl): delete it from current undo header, opportunistically?
-
   return true;
 }
 
@@ -202,9 +216,11 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
   }
 
   // the value is either zero or the lnum (row+1) if highlight was present.
-  static Map(uint64_t, uint64_t) *delete_set = NULL;
+  static Map(uint64_t, ssize_t) *delete_set = NULL;
+  typedef struct { Decoration *decor; int row1; } DecorItem;
+  static kvec_t(DecorItem) decors;
   if (delete_set == NULL) {
-    delete_set = map_new(uint64_t, uint64_t)();
+    delete_set = map_new(uint64_t, ssize_t)();
   }
 
   MarkTreeIter itr[1] = { 0 };
@@ -216,14 +232,16 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
         || (mark.row == u_row && mark.col > u_col)) {
       break;
     }
-    uint64_t *del_status = map_ref(uint64_t, uint64_t)(delete_set, mark.id,
+    ssize_t *del_status = map_ref(uint64_t, ssize_t)(delete_set, mark.id,
                                                      false);
     if (del_status) {
       marktree_del_itr(buf->b_marktree, itr, false);
-      map_del(uint64_t, uint64_t)(delete_set, mark.id);
+      if (*del_status >= 0) {  // we had a decor_id
-      if (*del_status > 0) {
+        DecorItem it = kv_A(decors, *del_status);
-        redraw_buf_range_later(buf, (linenr_T)(*del_status), mark.row+1);
+        decoration_redraw(buf, it.row1, mark.row, it.decor);
+        free_decoration(it.decor);
       }
+      map_del(uint64_t, ssize_t)(delete_set, mark.id);
       continue;
     }
 
@@ -233,15 +251,21 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
 
     assert(item.ns_id > 0 && item.mark_id > 0);
     if (item.mark_id > 0 && (item.ns_id == ns_id || all_ns)) {
-      if (kv_size(item.virt_text)) {
-        redraw_buf_line_later(buf, mark.row+1);
-      }
-      clear_virttext(&item.virt_text);
       marks_cleared = true;
       if (mark.id & MARKTREE_PAIRED_FLAG) {
         uint64_t other = mark.id ^ MARKTREE_END_FLAG;
-        uint64_t status = item.hl_id ? ((uint64_t)mark.row+1) : 0;
+        ssize_t decor_id = -1;
-        map_put(uint64_t, uint64_t)(delete_set, other, status);
+        if (item.decor) {
+          // Save the decoration and the first pos. Clear the decoration
+          // later when we know the full range.
+          decor_id = (ssize_t)kv_size(decors);
+          kv_push(decors,
+                  ((DecorItem) { .decor = item.decor, .row1 = mark.row }));
+        }
+        map_put(uint64_t, ssize_t)(delete_set, other, decor_id);
+      } else if (item.decor) {
+        decoration_redraw(buf, mark.row, mark.row, item.decor);
+        free_decoration(item.decor);
       }
       ExtmarkNs *my_ns = all_ns ? buf_ns_ref(buf, item.ns_id, false) : ns;
       map_del(uint64_t, uint64_t)(my_ns->map, item.mark_id);
@@ -251,16 +275,20 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
       marktree_itr_next(buf->b_marktree, itr);
     }
   }
-  uint64_t id, status;
+  uint64_t id;
-  map_foreach(delete_set, id, status, {
+  ssize_t decor_id;
+  map_foreach(delete_set, id, decor_id, {
     mtpos_t pos = marktree_lookup(buf->b_marktree, id, itr);
     assert(itr->node);
     marktree_del_itr(buf->b_marktree, itr, false);
-    if (status > 0) {
+    if (decor_id >= 0) {
-      redraw_buf_range_later(buf, (linenr_T)status, pos.row+1);
+      DecorItem it = kv_A(decors, decor_id);
+      decoration_redraw(buf, it.row1, pos.row, it.decor);
+      free_decoration(it.decor);
     }
   });
-  map_clear(uint64_t, uint64_t)(delete_set);
+  map_clear(uint64_t, ssize_t)(delete_set);
+  kv_size(decors) = 0;
   return marks_cleared;
 }
 
@@ -270,31 +298,44 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
 // will be searched to the start, or end
 // dir can be set to control the order of the array
 // amount = amount of marks to find or -1 for all
-ExtmarkArray extmark_get(buf_T *buf, uint64_t ns_id,
+ExtmarkInfoArray extmark_get(buf_T *buf, uint64_t ns_id,
                              int l_row, colnr_T l_col,
                              int u_row, colnr_T u_col,
                              int64_t amount, bool reverse)
 {
-  ExtmarkArray array = KV_INITIAL_VALUE;
+  ExtmarkInfoArray array = KV_INITIAL_VALUE;
-  MarkTreeIter itr[1] = { 0 };
+  MarkTreeIter itr[1];
   // Find all the marks
   marktree_itr_get_ext(buf->b_marktree, (mtpos_t){ l_row, l_col },
                        itr, reverse, false, NULL);
   int order = reverse ? -1 : 1;
   while ((int64_t)kv_size(array) < amount) {
     mtmark_t mark = marktree_itr_current(itr);
+    mtpos_t endpos = { -1, -1 };
     if (mark.row < 0
         || (mark.row - u_row) * order > 0
         || (mark.row == u_row && (mark.col - u_col) * order > 0)) {
       break;
     }
+    if (mark.id & MARKTREE_END_FLAG) {
+      goto next_mark;
+    } else if (mark.id & MARKTREE_PAIRED_FLAG) {
+      endpos = marktree_lookup(buf->b_marktree, mark.id | MARKTREE_END_FLAG,
+                               NULL);
+    }
+
+
     ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                       mark.id);
     if (item.ns_id == ns_id) {
       kv_push(array, ((ExtmarkInfo) { .ns_id = item.ns_id,
                                       .mark_id = item.mark_id,
-                                      .row = mark.row, .col = mark.col }));
+                                      .row = mark.row, .col = mark.col,
+                                      .end_row = endpos.row,
+                                      .end_col = endpos.col,
+                                      .decor = item.decor }));
     }
+next_mark:
     if (reverse) {
       marktree_itr_prev(buf->b_marktree, itr);
     } else {
@@ -308,7 +349,7 @@ ExtmarkArray extmark_get(buf_T *buf, uint64_t ns_id,
 ExtmarkInfo extmark_from_id(buf_T *buf, uint64_t ns_id, uint64_t id)
 {
   ExtmarkNs *ns = buf_ns_ref(buf, ns_id, false);
-  ExtmarkInfo ret = { 0, 0, -1, -1 };
+  ExtmarkInfo ret = { 0, 0, -1, -1, -1, -1, NULL };
   if (!ns) {
     return ret;
   }
@@ -319,12 +360,22 @@ ExtmarkInfo extmark_from_id(buf_T *buf, uint64_t ns_id, uint64_t id)
   }
 
   mtpos_t pos = marktree_lookup(buf->b_marktree, mark, NULL);
+  mtpos_t endpos = { -1, -1 };
+  if (mark & MARKTREE_PAIRED_FLAG) {
+    endpos = marktree_lookup(buf->b_marktree, mark | MARKTREE_END_FLAG, NULL);
+  }
   assert(pos.row >= 0);
 
+  ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
+                                                    mark);
+
   ret.ns_id = ns_id;
   ret.mark_id = id;
   ret.row = pos.row;
   ret.col = pos.col;
+  ret.end_row = endpos.row;
+  ret.end_col = endpos.col;
+  ret.decor = item.decor;
 
   return ret;
 }
@@ -352,7 +403,7 @@ void extmark_free_all(buf_T *buf)
 
   map_foreach(buf->b_extmark_index, id, item, {
     (void)id;
-    clear_virttext(&item.virt_text);
+    free_decoration(item.decor);
   });
   map_free(uint64_t, ExtmarkItem)(buf->b_extmark_index);
   buf->b_extmark_index = NULL;
@@ -642,50 +693,6 @@ uint64_t src2ns(Integer *src_id)
   }
 }
 
-/// Adds a decoration to a buffer.
-///
-/// Unlike matchaddpos() highlights, these follow changes to the the buffer
-/// texts. Decorations are represented internally and in the API as extmarks.
-///
-/// @param buf The buffer to add decorations to
-/// @param ns_id A valid namespace id.
-/// @param hl_id Id of the highlight group to use (or zero)
-/// @param start_row The line to highlight
-/// @param start_col First column to highlight
-/// @param end_row The line to highlight
-/// @param end_col The last column to highlight
-/// @param virt_text Virtual text (currently placed at the EOL of start_row)
-/// @return The extmark id inside the namespace
-uint64_t extmark_add_decoration(buf_T *buf, uint64_t ns_id, int hl_id,
-                                int start_row, colnr_T start_col,
-                                int end_row, colnr_T end_col,
-                                VirtText virt_text)
-{
-  ExtmarkNs *ns = buf_ns_ref(buf, ns_id, true);
-  ExtmarkItem item;
-  item.ns_id = ns_id;
-  item.mark_id = ns->free_id++;
-  item.hl_id = hl_id;
-  item.virt_text = virt_text;
-
-  uint64_t mark;
-
-  if (end_row > -1) {
-    mark = marktree_put_pair(buf->b_marktree,
-                             start_row, start_col, true,
-                             end_row, end_col, false);
-  } else {
-    mark = marktree_put(buf->b_marktree, start_row, start_col, true);
-  }
-
-  map_put(uint64_t, ExtmarkItem)(buf->b_extmark_index, mark, item);
-  map_put(uint64_t, uint64_t)(ns->map, item.mark_id, mark);
-
-  redraw_buf_range_later(buf, start_row+1,
-                         (end_row >= 0 ? end_row : start_row) + 1);
-  return item.mark_id;
-}
-
 /// Add highlighting to a buffer, bounded by two cursor positions,
 /// with an offset.
 ///
@@ -705,6 +712,7 @@ void bufhl_add_hl_pos_offset(buf_T *buf,
 {
   colnr_T hl_start = 0;
   colnr_T hl_end = 0;
+  Decoration *decor = decoration_hl(hl_id);
 
   // TODO(bfredl): if decoration had blocky mode, we could avoid this loop
   for (linenr_T lnum = pos_start.lnum; lnum <= pos_end.lnum; lnum ++) {
@@ -729,10 +737,44 @@ void bufhl_add_hl_pos_offset(buf_T *buf,
       hl_start = pos_start.col + offset;
       hl_end = pos_end.col + offset;
     }
-    (void)extmark_add_decoration(buf, (uint64_t)src_id, hl_id,
+    (void)extmark_set(buf, (uint64_t)src_id, 0,
-                                 (int)lnum-1, hl_start,
+                      (int)lnum-1, hl_start, (int)lnum-1+end_off, hl_end,
-                                 (int)lnum-1+end_off, hl_end,
+                      decor, kExtmarkUndo);
-                                 VIRTTEXT_EMPTY);
+  }
+}
+
+Decoration *decoration_hl(int hl_id)
+{
+  assert(hl_id > 0);
+  Decoration **dp = (Decoration **)pmap_ref(uint64_t)(hl_decors,
+                                                      (uint64_t)hl_id, true);
+  if (*dp) {
+    return *dp;
+  }
+
+  Decoration *decor = xcalloc(1, sizeof(*decor));
+  decor->hl_id = hl_id;
+  decor->shared = true;
+  *dp = decor;
+  return decor;
+}
+
+void decoration_redraw(buf_T *buf, int row1, int row2, Decoration *decor)
+{
+  if (decor->hl_id && row2 >= row1) {
+    redraw_buf_range_later(buf, row1+1, row2+1);
+  }
+
+  if (kv_size(decor->virt_text)) {
+    redraw_buf_line_later(buf, row1+1);
+  }
+}
+
+void free_decoration(Decoration *decor)
+{
+  if (decor && !decor->shared) {
+    clear_virttext(&decor->virt_text);
+    xfree(decor);
   }
 }
 
@@ -757,8 +799,8 @@ VirtText *extmark_find_virttext(buf_T *buf, int row, uint64_t ns_id)
     ExtmarkItem *item = map_ref(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                        mark.id, false);
     if (item && (ns_id == 0 || ns_id == item->ns_id)
-        && kv_size(item->virt_text)) {
+        && item->decor && kv_size(item->decor->virt_text)) {
-      return &item->virt_text;
+      return &item->decor->virt_text;
     }
     marktree_itr_next(buf->b_marktree, itr);
   }
@@ -787,7 +829,6 @@ bool decorations_redraw_start(buf_T *buf, int top_row,
     if (mark.row < 0) {  // || mark.row > end_row
       break;
     }
-    // TODO(bfredl): dedicated flag for being a decoration?
     if ((mark.row < top_row && mark.id&MARKTREE_END_FLAG)) {
       goto next_mark;
     }
@@ -797,15 +838,20 @@ bool decorations_redraw_start(buf_T *buf, int top_row,
     uint64_t start_id = mark.id & ~MARKTREE_END_FLAG;
     ExtmarkItem *item = map_ref(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                        start_id, false);
+    if (!item || !item->decor) {
+    // TODO(bfredl): dedicated flag for being a decoration?
+      goto next_mark;
+    }
+    Decoration *decor = item->decor;
+
     if ((!(mark.id&MARKTREE_END_FLAG) && altpos.row < top_row
-         && item && !kv_size(item->virt_text))
+         && item && !kv_size(decor->virt_text))
         || ((mark.id&MARKTREE_END_FLAG) && altpos.row >= top_row)) {
       goto next_mark;
     }
 
-    if (item && (item->hl_id > 0 || kv_size(item->virt_text))) {
+    int attr_id = decor->hl_id > 0 ? syn_id2attr(decor->hl_id) : 0;
-      int attr_id = item->hl_id > 0 ? syn_id2attr(item->hl_id) : 0;
+    VirtText *vt = kv_size(decor->virt_text) ? &decor->virt_text : NULL;
-      VirtText *vt = kv_size(item->virt_text) ? &item->virt_text : NULL;
     HlRange range;
     if (mark.id&MARKTREE_END_FLAG) {
       range = (HlRange){ altpos.row, altpos.col, mark.row, mark.col,
@@ -815,7 +861,7 @@ bool decorations_redraw_start(buf_T *buf, int top_row,
                          altpos.col, attr_id, vt };
     }
     kv_push(state->active, range);
-    }
+
 next_mark:
     if (marktree_itr_node_done(state->itr)) {
       break;
@@ -860,21 +906,24 @@ int decorations_redraw_col(buf_T *buf, int col, DecorationRedrawState *state)
 
     ExtmarkItem *item = map_ref(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                        mark.id, false);
+    if (!item || !item->decor) {
+    // TODO(bfredl): dedicated flag for being a decoration?
+      goto next_mark;
+    }
+    Decoration *decor = item->decor;
 
     if (endpos.row < mark.row
         || (endpos.row == mark.row && endpos.col <= mark.col)) {
-      if (item && !kv_size(item->virt_text)) {
+      if (item && !kv_size(decor->virt_text)) {
         goto next_mark;
       }
     }
 
-    if (item && (item->hl_id > 0 || kv_size(item->virt_text))) {
+    int attr_id = decor->hl_id > 0 ? syn_id2attr(decor->hl_id) : 0;
-      int attr_id = item->hl_id > 0 ? syn_id2attr(item->hl_id) : 0;
+    VirtText *vt = kv_size(decor->virt_text) ? &decor->virt_text : NULL;
-      VirtText *vt = kv_size(item->virt_text) ? &item->virt_text : NULL;
     kv_push(state->active, ((HlRange){ mark.row, mark.col,
                                        endpos.row, endpos.col,
                                        attr_id, vt }));
-    }
 
 next_mark:
     marktree_itr_next(buf->b_marktree, state->itr);

src/nvim/extmark.h

@@ -13,9 +13,12 @@ typedef struct
   uint64_t mark_id;
   int row;
   colnr_T col;
+  int end_row;
+  colnr_T end_col;
+  Decoration *decor;
 } ExtmarkInfo;
 
-typedef kvec_t(ExtmarkInfo) ExtmarkArray;
+typedef kvec_t(ExtmarkInfo) ExtmarkInfoArray;
 
 
 // delete the columns between mincol and endcol

src/nvim/extmark_defs.h

@@ -12,14 +12,22 @@ typedef struct {
 typedef kvec_t(VirtTextChunk) VirtText;
 #define VIRTTEXT_EMPTY ((VirtText)KV_INITIAL_VALUE)
 
+typedef struct
+{
+  int hl_id;  // highlight group
+  VirtText virt_text;
+  // TODO(bfredl): style, signs, etc
+  bool shared;  // shared decoration, don't free
+} Decoration;
+
 typedef struct
 {
   uint64_t ns_id;
   uint64_t mark_id;
-  int hl_id;  // highlight group
+  // TODO(bfredl): a lot of small allocations. Should probably use
-  // TODO(bfredl): virt_text is pretty larger than the rest,
+  // kvec_t(Decoration) as an arena. Alternatively, store ns_id/mark_id
-  // pointer indirection?
+  // _inline_ in MarkTree and use the map only for decorations.
-  VirtText virt_text;
+  Decoration *decor;
 } ExtmarkItem;
 
 typedef struct undo_object ExtmarkUndoObject;

src/nvim/main.c

@@ -21,6 +21,7 @@
 #include "nvim/ex_cmds.h"
 #include "nvim/ex_cmds2.h"
 #include "nvim/ex_docmd.h"
+#include "nvim/extmark.h"
 #include "nvim/fileio.h"
 #include "nvim/fold.h"
 #include "nvim/getchar.h"
@@ -160,6 +161,7 @@ void early_init(mparm_T *paramp)
   env_init();
   fs_init();
   handle_init();
+  extmark_init();
   eval_init();          // init global variables
   init_path(argv0 ? argv0 : "nvim");
   init_normal_cmds();   // Init the table of Normal mode commands.

src/nvim/map.c

@@ -184,7 +184,7 @@ MAP_IMPL(uint64_t, uint64_t, DEFAULT_INITIALIZER)
 #define EXTMARK_NS_INITIALIZER { 0, 0 }
 MAP_IMPL(uint64_t, ExtmarkNs, EXTMARK_NS_INITIALIZER)
 #define KVEC_INITIALIZER { .size = 0, .capacity = 0, .items = NULL }
-#define EXTMARK_ITEM_INITIALIZER { 0, 0, 0, KVEC_INITIALIZER }
+#define EXTMARK_ITEM_INITIALIZER { 0, 0, NULL }
 MAP_IMPL(uint64_t, ExtmarkItem, EXTMARK_ITEM_INITIALIZER)
 MAP_IMPL(handle_T, ptr_t, DEFAULT_INITIALIZER)
 #define MSGPACK_HANDLER_INITIALIZER { .fn = NULL, .fast = false }

src/nvim/map.h

@@ -73,6 +73,7 @@ MAP_DECLS(String, handle_T)
 #define pmap_has(T) map_has(T, ptr_t)
 #define pmap_key(T) map_key(T, ptr_t)
 #define pmap_put(T) map_put(T, ptr_t)
+#define pmap_ref(T) map_ref(T, ptr_t)
 /// @see pmap_del2
 #define pmap_del(T) map_del(T, ptr_t)
 #define pmap_clear(T) map_clear(T, ptr_t)

src/nvim/screen.c

@@ -2554,6 +2554,7 @@ win_line (
   }
 
   // If this line has a sign with line highlighting set line_attr.
+  // TODO(bfredl, vigoux): this should not take priority over decorations!
   v = buf_getsigntype(wp->w_buffer, lnum, SIGN_LINEHL, 0, 1);
   if (v != 0) {
     line_attr = sign_get_attr((int)v, SIGN_LINEHL);

test/functional/api/extmark_spec.lua

@@ -18,13 +18,13 @@ local function expect(contents)
 end
 
 local function check_undo_redo(ns, mark, sr, sc, er, ec) --s = start, e = end
-  local rv = curbufmeths.get_extmark_by_id(ns, mark)
+  local rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({er, ec}, rv)
   feed("u")
-  rv = curbufmeths.get_extmark_by_id(ns, mark)
+  rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({sr, sc}, rv)
   feed("<c-r>")
-  rv = curbufmeths.get_extmark_by_id(ns, mark)
+  rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({er, ec}, rv)
 end
 
@@ -32,14 +32,17 @@ local function set_extmark(ns_id, id, line, col, opts)
   if opts == nil then
     opts = {}
   end
-  return curbufmeths.set_extmark(ns_id, id, line, col, opts)
+  if id ~= nil and id ~= 0 then
+    opts.id = id
+  end
+  return curbufmeths.set_extmark(ns_id, line, col, opts)
 end
 
 local function get_extmarks(ns_id, start, end_, opts)
   if opts == nil then
     opts = {}
   end
-  return curbufmeths.get_extmarks(ns_id, start, end_, opts)
+  return curbufmeths.get_extmarks(ns_id, start, end_, opts, false)
 end
 
 local function batch_set(ns_id, positions)
@@ -93,7 +96,7 @@ describe('API/extmarks', function()
   it('adds, updates  and deletes marks', function()
     local rv = set_extmark(ns, marks[1], positions[1][1], positions[1][2])
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({positions[1][1], positions[1][2]}, rv)
     -- Test adding a second mark on same row works
     rv = set_extmark(ns, marks[2], positions[2][1], positions[2][2])
@@ -102,14 +105,14 @@ describe('API/extmarks', function()
     -- Test an update, (same pos)
     rv = set_extmark(ns, marks[1], positions[1][1], positions[1][2])
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[2])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[2], false)
     eq({positions[2][1], positions[2][2]}, rv)
     -- Test an update, (new pos)
     row = positions[1][1]
     col = positions[1][2] + 1
     rv = set_extmark(ns, marks[1], row, col)
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({row, col}, rv)
 
     -- remove the test marks
@@ -432,7 +435,7 @@ describe('API/extmarks', function()
       ~              |
                      |
     ]])
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 6}, rv)
     check_undo_redo(ns, marks[1], 0, 3, 0, 6)
   end)
@@ -906,9 +909,9 @@ describe('API/extmarks', function()
     -- Set the mark before the cursor, should stay there
     set_extmark(ns, marks[2], 0, 10)
     feed("i<cr><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[2])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[2], false)
     eq({0, 10}, rv)
     check_undo_redo(ns, marks[1], 0, 12, 1, 3)
   end)
@@ -921,12 +924,12 @@ describe('API/extmarks', function()
     feed("0iint <esc>A {<cr><esc>0i1M1<esc>")
     set_extmark(ns, marks[1], 1, 1)
     feed("0i<c-f><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
     check_undo_redo(ns, marks[1], 1, 1, 1, 3)
     -- now check when cursor at eol
     feed("uA<c-f><esc>")
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
   end)
 
@@ -937,12 +940,12 @@ describe('API/extmarks', function()
     feed("0i<tab><esc>")
     set_extmark(ns, marks[1], 0, 3)
     feed("bi<c-d><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 1}, rv)
     check_undo_redo(ns, marks[1], 0, 3, 0, 1)
     -- check when cursor at eol
     feed("uA<c-d><esc>")
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 1}, rv)
   end)
 
@@ -1072,7 +1075,7 @@ describe('API/extmarks', function()
     check_undo_redo(ns, marks[5], 2, 0, 3, 0)
     feed('u')
     feed([[:1,2s:3:\rxx<cr>]])
-    eq({1, 3}, curbufmeths.get_extmark_by_id(ns, marks[3]))
+    eq({1, 3}, curbufmeths.get_extmark_by_id(ns, marks[3], false))
   end)
 
   it('substitions over multiple lines with replace in substition', function()
@@ -1311,16 +1314,16 @@ describe('API/extmarks', function()
     eq("Invalid ns_id", pcall_err(set_extmark, ns_invalid, marks[1], positions[1][1], positions[1][2]))
     eq("Invalid ns_id", pcall_err(curbufmeths.del_extmark, ns_invalid, marks[1]))
     eq("Invalid ns_id", pcall_err(get_extmarks, ns_invalid, positions[1], positions[2]))
-    eq("Invalid ns_id", pcall_err(curbufmeths.get_extmark_by_id, ns_invalid, marks[1]))
+    eq("Invalid ns_id", pcall_err(curbufmeths.get_extmark_by_id, ns_invalid, marks[1], false))
   end)
 
   it('when col = line-length, set the mark on eol', function()
     set_extmark(ns, marks[1], 0, -1)
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, init_text:len()}, rv)
     -- Test another
     set_extmark(ns, marks[1], 0, -1)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, init_text:len()}, rv)
   end)
 
@@ -1333,7 +1336,7 @@ describe('API/extmarks', function()
     local invalid_col = init_text:len() + 1
     local invalid_lnum = 3
     eq('line value outside range', pcall_err(set_extmark, ns, marks[1], invalid_lnum, invalid_col))
-    eq({}, curbufmeths.get_extmark_by_id(ns, marks[1]))
+    eq({}, curbufmeths.get_extmark_by_id(ns, marks[1], false))
   end)
 
   it('bug from check_col in extmark_set', function()
@@ -1357,14 +1360,14 @@ describe('API/extmarks', function()
   it('can set a mark to other buffer', function()
     local buf = request('nvim_create_buf', 0, 1)
     request('nvim_buf_set_lines', buf, 0, -1, 1, {"", ""})
-    local id = bufmeths.set_extmark(buf, ns, 0, 1, 0, {})
+    local id = bufmeths.set_extmark(buf, ns, 1, 0, {})
-    eq({{id, 1, 0}}, bufmeths.get_extmarks(buf, ns, 0, -1, {}))
+    eq({{id, 1, 0}}, bufmeths.get_extmarks(buf, ns, 0, -1, {}, false))
   end)
 
   it('does not crash with append/delete/undo seqence', function()
      meths.exec([[
       let ns = nvim_create_namespace('myplugin')
-      call nvim_buf_set_extmark(0, ns, 0, 0, 0, {})
+      call nvim_buf_set_extmark(0, ns, 0, 0, {})
       call append(0, '')
       %delete
       undo]],false)

test/functional/ui/bufhl_spec.lua

@@ -690,7 +690,7 @@ describe('Buffer highlighting', function()
     end)
 
     it('can be retrieved', function()
-      local get_virtual_text = curbufmeths.get_virtual_text
+      local get_extmarks = curbufmeths.get_extmarks
       local line_count = curbufmeths.line_count
 
       local s1 = {{'Köttbullar', 'Comment'}, {'Kräuterbutter'}}
@@ -699,12 +699,14 @@ describe('Buffer highlighting', function()
       -- TODO: only a virtual text from the same ns curretly overrides
       -- an existing virtual text. We might add a prioritation system.
       set_virtual_text(id1, 0, s1, {})
-      eq(s1, get_virtual_text(0))
+      eq({{1, 0, 0, {virt_text = s1}}}, get_extmarks(id1, {0,0}, {0, -1}, {}, true))
 
-      set_virtual_text(-1, line_count(), s2, {})
+      -- TODO: is this really valid? shouldn't the max be line_count()-1?
-      eq(s2, get_virtual_text(line_count()))
+      local lastline = line_count()
+      set_virtual_text(id1, line_count(), s2, {})
+      eq({{3, lastline, 0, {virt_text = s2}}}, get_extmarks(id1, {lastline,0}, {lastline, -1}, {}, true))
 
-      eq({}, get_virtual_text(line_count() + 9000))
+      eq({}, get_extmarks(id1, {lastline+9000,0}, {lastline+9000, -1}, {}, false))
     end)
 
     it('is not highlighted by visual selection', function()