fix(treesitter): use subpriorities for tree ordering

This partially reverts 0b8a72b739,
that is unreverts 15e77a56b7

"priority" is an internal neovim concept which does not occur in shared
queries. Ideally a single priority space should eventually be enough
for our needs. But as we don't want to poke at the usages of
priorities right now in the wider ecosystem,
introduce the "subpriorities" so that treesitter code can distinguish
highlights of the same priorities with different tree nesting depth.

This mainly affects `injection.combined` as parent-tree nodes might appear
in the middle of child-tree nodes which otherwise is not possible.

runtime/lua/vim/_meta/api_keysets.lua

@@ -423,6 +423,7 @@ error('Cannot require a meta file')
 --- @field undo_restore? boolean
 --- @field url? string
 --- @field scoped? boolean
+--- @field _subpriority? integer
 
 --- @class vim.api.keyset.user_command
 --- @field addr? any

runtime/lua/vim/treesitter/highlighter.lua

@@ -360,7 +360,10 @@ local function on_range_impl(
   local MAX_ROW = 2147483647 -- sentinel for skipping to the end of file
   local skip_until_row = MAX_ROW
   local skip_until_col = 0
+
+  local subtree_counter = 0
   self:for_each_highlight_state(win, function(state)
+    subtree_counter = subtree_counter + 1
     local root_node = state.tstree:root()
     ---@type { [1]: integer, [2]: integer, [3]: integer, [4]: integer }
     local root_range = { root_node:range() }
@@ -449,6 +452,7 @@ local function on_range_impl(
               conceal = conceal,
               spell = spell,
               url = url,
+              _subpriority = subtree_counter,
             })
           end
 

src/nvim/api/extmark.c

@@ -836,6 +836,15 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
       col2 = c;
     }
 
+    DecorPriority subpriority = 0;
+    if (HAS_KEY(opts, set_extmark, _subpriority)) {
+      VALIDATE_RANGE((opts->_subpriority >= 0 && opts->_subpriority <= UINT16_MAX),
+                     "_subpriority", {
+        goto error;
+      });
+      subpriority = (DecorPriority)opts->_subpriority;
+    }
+
     if (kv_size(virt_text.data.virt_text)) {
       decor_range_add_virt(&decor_state, r, c, line2, col2, decor_put_vt(virt_text, NULL), true);
     }
@@ -845,7 +854,8 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id, Integer line, Integer
     if (has_hl) {
       DecorSignHighlight sh = decor_sh_from_inline(hl);
       sh.url = url;
-      decor_range_add_sh(&decor_state, r, c, line2, col2, &sh, true, (uint32_t)ns_id, id);
+      decor_range_add_sh(&decor_state, r, c, line2, col2, &sh, true, (uint32_t)ns_id, id,
+                         subpriority);
     }
   } else {
     if (opts->ephemeral) {

src/nvim/api/keysets_defs.h

@@ -62,6 +62,8 @@ typedef struct {
   Boolean undo_restore;
   String url;
   Boolean scoped;
+
+  Integer _subpriority;
 } Dict(set_extmark);
 
 typedef struct {

src/nvim/decoration.c

@@ -553,12 +553,12 @@ static void decor_range_add_from_inline(DecorState *state, int start_row, int st
     uint32_t idx = decor.data.ext.sh_idx;
     while (idx != DECOR_ID_INVALID) {
       DecorSignHighlight *sh = &kv_A(decor_items, idx);
-      decor_range_add_sh(state, start_row, start_col, end_row, end_col, sh, owned, ns, mark_id);
+      decor_range_add_sh(state, start_row, start_col, end_row, end_col, sh, owned, ns, mark_id, 0);
       idx = sh->next;
     }
   } else {
     DecorSignHighlight sh = decor_sh_from_inline(decor.data.hl);
-    decor_range_add_sh(state, start_row, start_col, end_row, end_col, &sh, owned, ns, mark_id);
+    decor_range_add_sh(state, start_row, start_col, end_row, end_col, &sh, owned, ns, mark_id, 0);
   }
 }
 
@@ -619,14 +619,15 @@ void decor_range_add_virt(DecorState *state, int start_row, int start_col, int e
     .data.vt = vt,
     .attr_id = 0,
     .owned = owned,
-    .priority = vt->priority,
+    .priority_internal = ((DecorPriorityInternal)vt->priority << 16),
     .draw_col = -10,
   };
   decor_range_insert(state, &range);
 }
 
 void decor_range_add_sh(DecorState *state, int start_row, int start_col, int end_row, int end_col,
-                        DecorSignHighlight *sh, bool owned, uint32_t ns, uint32_t mark_id)
+                        DecorSignHighlight *sh, bool owned, uint32_t ns, uint32_t mark_id,
+                        DecorPriority subpriority)
 {
   if (sh->flags & kSHIsSign) {
     return;
@@ -638,7 +639,7 @@ void decor_range_add_sh(DecorState *state, int start_row, int start_col, int end
     .data.sh = *sh,
     .attr_id = 0,
     .owned = owned,
-    .priority = sh->priority,
+    .priority_internal = ((DecorPriorityInternal)sh->priority << 16) + subpriority,
     .draw_col = -10,
   };
 
@@ -731,7 +732,7 @@ next_mark:
       break;
     }
     int const ordering = r->ordering;
-    DecorPriority const priority = r->priority;
+    DecorPriorityInternal const priority = r->priority_internal;
 
     int begin = 0;
     int end = cur_end;
@@ -739,7 +740,8 @@ next_mark:
       int mid = begin + ((end - begin) >> 1);
       int mi = indices[mid];
       DecorRange *mr = &slots[mi].range;
-      if (mr->priority < priority || (mr->priority == priority && mr->ordering < ordering)) {
+      if (mr->priority_internal < priority
+          || (mr->priority_internal == priority && mr->ordering < ordering)) {
         begin = mid + 1;
       } else {
         end = mid;

src/nvim/decoration.h

@@ -37,7 +37,7 @@ typedef struct {
   int end_row;
   int end_col;
   int ordering;  ///< range insertion order
-  DecorPriority priority;
+  DecorPriorityInternal priority_internal;
   bool owned;  ///< ephemeral decoration, free memory immediately
   DecorRangeKind kind;
   // next pointers MUST NOT be used, these are separate ranges

src/nvim/decoration_defs.h

@@ -36,6 +36,7 @@ enum {
 typedef kvec_t(struct virt_line { VirtText line; int flags; }) VirtLines;
 
 typedef uint16_t DecorPriority;
+typedef uint32_t DecorPriorityInternal;
 #define DECOR_PRIORITY_BASE 0x1000
 
 /// Keep in sync with hl_mode_str[] in decoration.h

test/functional/treesitter/highlight_spec.lua

@@ -548,7 +548,7 @@ describe('treesitter highlighting (C)', function()
           lua = [[
             ; query
             (string) @string
-            ((comment) @comment (#set! priority 90))
+            (comment) @comment
             (function_call (identifier) @function.call)
             [ "(" ")" ] @punctuation.bracket
           ]],