treesitter: use new on_bytes interface

This will significantly reduce the parsing work
needed e.g. when rehighlighting after every keypress
in insert mode.

Also add safety check for tree-sitter trying to read
past the end of a line. This can happen after we sent
an incorrect buffer update.

runtime/lua/vim/treesitter.lua

@@ -33,16 +33,23 @@ function Parser:parse()
   return self.tree, changes
 end
 
-function Parser:_on_lines(bufnr, changed_tick, start_row, old_stop_row, stop_row, old_byte_size)
+function Parser:_on_bytes(bufnr, changed_tick,
-  local start_byte = a.nvim_buf_get_offset(bufnr,start_row)
+                          start_row, start_col, start_byte,
-  local stop_byte = a.nvim_buf_get_offset(bufnr,stop_row)
+                          old_row, old_col, old_byte,
-  local old_stop_byte = start_byte + old_byte_size
+                          new_row, new_col, new_byte)
-  self._parser:edit(start_byte,old_stop_byte,stop_byte,
+  local old_end_col = old_col + ((old_row == 0) and start_col or 0)
-                    start_row,0,old_stop_row,0,stop_row,0)
+  local new_end_col = new_col + ((new_row == 0) and start_col or 0)
+  self._parser:edit(start_byte,start_byte+old_byte,start_byte+new_byte,
+                    start_row, start_col,
+                    start_row+old_row, old_end_col,
+                    start_row+new_row, new_end_col)
   self.valid = false
 
-  for _, cb in ipairs(self.lines_cbs) do
+  for _, cb in ipairs(self.bytes_cbs) do
-    cb(bufnr, changed_tick, start_row, old_stop_row, stop_row, old_byte_size)
+    cb(bufnr, changed_tick,
+      start_row, start_col, start_byte,
+      old_row, old_col, old_byte,
+      new_row, new_col, new_byte)
   end
 end
 
@@ -88,12 +95,12 @@ function M._create_parser(bufnr, lang, id)
   local self = setmetatable({bufnr=bufnr, lang=lang, valid=false}, Parser)
   self._parser = vim._create_ts_parser(lang)
   self.changedtree_cbs = {}
-  self.lines_cbs = {}
+  self.bytes_cbs = {}
   self:parse()
-  -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
+    -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
-  -- using it.
+    -- using it.
-  local function lines_cb(_, ...)
+  local function bytes_cb(_, ...)
-    return self:_on_lines(...)
+    return self:_on_bytes(...)
   end
   local detach_cb = nil
   if id ~= nil then
@@ -103,7 +110,7 @@ function M._create_parser(bufnr, lang, id)
       end
     end
   end
-  a.nvim_buf_attach(self.bufnr, false, {on_lines=lines_cb, on_detach=detach_cb})
+  a.nvim_buf_attach(self.bufnr, false, {on_bytes=bytes_cb, on_detach=detach_cb})
   return self
 end
 
@@ -138,8 +145,8 @@ function M.get_parser(bufnr, lang, buf_attach_cbs)
     table.insert(parsers[id].changedtree_cbs, buf_attach_cbs.on_changedtree)
   end
 
-  if buf_attach_cbs and buf_attach_cbs.on_lines then
+  if buf_attach_cbs and buf_attach_cbs.on_bytes then
-    table.insert(parsers[id].lines_cbs, buf_attach_cbs.on_lines)
+    table.insert(parsers[id].bytes_cbs, buf_attach_cbs.on_bytes)
   end
 
   return parsers[id]

runtime/lua/vim/treesitter/highlighter.lua

@@ -60,7 +60,7 @@ function TSHighlighter.new(query, bufnr, ft)
     ft,
     {
       on_changedtree = function(...) self:on_changedtree(...) end,
-      on_lines = function() self.root = self.parser:parse():root() end
+      on_bytes = function() self.root = self.parser:parse():root() end
     }
   )
 

src/nvim/lua/treesitter.c

@@ -286,22 +286,21 @@ static const char *input_cb(void *payload, uint32_t byte_index,
   }
   char_u *line = ml_get_buf(bp, position.row+1, false);
   size_t len = STRLEN(line);
-
   if (position.column > len) {
     *bytes_read = 0;
-  } else {
+    return "";
-    size_t tocopy = MIN(len-position.column, BUFSIZE);
+  }
+  size_t tocopy = MIN(len-position.column, BUFSIZE);
 
-    memcpy(buf, line+position.column, tocopy);
+  memcpy(buf, line+position.column, tocopy);
-    // Translate embedded \n to NUL
+  // Translate embedded \n to NUL
-    memchrsub(buf, '\n', '\0', tocopy);
+  memchrsub(buf, '\n', '\0', tocopy);
-    *bytes_read = (uint32_t)tocopy;
+  *bytes_read = (uint32_t)tocopy;
-    if (tocopy < BUFSIZE) {
+  if (tocopy < BUFSIZE) {
-      // now add the final \n. If it didn't fit, input_cb will be called again
+    // now add the final \n. If it didn't fit, input_cb will be called again
-      // on the same line with advanced column.
+    // on the same line with advanced column.
-      buf[tocopy] = '\n';
+    buf[tocopy] = '\n';
-      (*bytes_read)++;
+    (*bytes_read)++;
-    }
   }
   return buf;
 #undef BUFSIZE