lua: add {old_byte_size} to on_lines buffer change event

runtime/doc/api.txt

@@ -200,17 +200,23 @@ User reloads the buffer with ":edit", emits: >
   nvim_buf_detach_event[{buf}]
 
                                                         *api-buffer-updates-lua*
-In-process lua plugins can also recieve buffer updates, in the form of lua
+In-process lua plugins can also receive buffer updates, in the form of lua
 callbacks. These callbacks are called frequently in various contexts, buffer
 contents or window layout should not be changed inside these |textlock|.
 |vim.schedule| can be used to defer these operations to the main loop, where
 they are allowed.
 
 |nvim_buf_attach| will take keyword args for the callbacks. "on_lines" will
-receive parameters ("lines", {buf}, {changedtick}, {firstline}, {lastline}, {new_lastline}).
+receive parameters ("lines", {buf}, {changedtick}, {firstline}, {lastline},
-Unlike remote channels the text contents are not passed. The new text can be
+{new_lastline}, {old_bytecount}).
-accessed inside the callback as
+Unlike remote channel events the text contents are not passed. The new text can
-`vim.api.nvim_buf_get_lines(buf, firstline, new_lastline, true)`
+be accessed inside the callback as
+
+    `vim.api.nvim_buf_get_lines(buf, firstline, new_lastline, true)`
+
+{old_bytecount} is the total size of the replaced region {firstline} to
+{lastline} in bytes, including the final newline after {lastline}.
+
 "on_changedtick" is invoked when |b:changedtick| was incremented but no text
 was changed. The parameters recieved are ("changedtick", {buf}, {changedtick}).
 

src/nvim/api/buffer.c

@@ -1176,6 +1176,29 @@ free_exit:
   return 0;
 }
 
+Dictionary nvim__buf_stats(Buffer buffer, Error *err)
+{
+  Dictionary rv = ARRAY_DICT_INIT;
+
+  buf_T *buf = find_buffer_by_handle(buffer, err);
+  if (!buf) {
+    return rv;
+  }
+
+  // Number of times the cached line was flushed.
+  // This should generally not increase while editing the same
+  // line in the same mode.
+  PUT(rv, "flush_count", INTEGER_OBJ(buf->flush_count));
+  // lnum of current line
+  PUT(rv, "current_lnum", INTEGER_OBJ(buf->b_ml.ml_line_lnum));
+  // whether the line has unflushed changes.
+  PUT(rv, "line_dirty", BOOLEAN_OBJ(buf->b_ml.ml_flags & ML_LINE_DIRTY));
+  // NB: this should be zero at any time API functions are called,
+  // this exists to debug issues
+  PUT(rv, "dirty_bytes", INTEGER_OBJ((Integer)buf->deleted_bytes));
+  return rv;
+}
+
 // Check if deleting lines made the cursor position invalid.
 // Changed lines from `lo` to `hi`; added `extra` lines (negative if deleted).
 static void fix_cursor(linenr_T lo, linenr_T hi, linenr_T extra)

src/nvim/buffer_defs.h

@@ -807,6 +807,9 @@ struct file_buffer {
   kvec_t(uint64_t) update_channels;
   kvec_t(BufUpdateCallbacks) update_callbacks;
 
+  size_t deleted_bytes;
+  int flush_count;
+
   int b_diff_failed;    // internal diff failed for this buffer
 };
 

src/nvim/buffer_updates.c

@@ -169,6 +169,8 @@ void buf_updates_send_changes(buf_T *buf,
                               int64_t num_removed,
                               bool send_tick)
 {
+  size_t deleted_bytes = ml_flush_deleted_bytes(buf);
+
   if (!buf_updates_active(buf)) {
     return;
   }
@@ -231,8 +233,8 @@ void buf_updates_send_changes(buf_T *buf,
     bool keep = true;
     if (cb.on_lines != LUA_NOREF) {
       Array args = ARRAY_DICT_INIT;
-      Object items[5];
+      Object items[6];
-      args.size = 5;
+      args.size = 6;
       args.items = items;
 
       // the first argument is always the buffer handle
@@ -250,6 +252,8 @@ void buf_updates_send_changes(buf_T *buf,
       // the last line in the updated range
       args.items[4] = INTEGER_OBJ(firstline - 1 + num_added);
 
+      // byte count of previous contents
+      args.items[5] = INTEGER_OBJ((Integer)deleted_bytes);
       textlock++;
       Object res = executor_exec_lua_cb(cb.on_lines, "lines", args, true);
       textlock--;

src/nvim/fileio.c

@@ -1755,6 +1755,7 @@ failed:
       ml_delete(curbuf->b_ml.ml_line_count, false);
       linecnt--;
     }
+    curbuf->deleted_bytes = 0;
     linecnt = curbuf->b_ml.ml_line_count - linecnt;
     if (filesize == 0)
       linecnt = 0;

src/nvim/memline.c

@@ -2403,13 +2403,26 @@ int ml_replace(linenr_T lnum, char_u *line, bool copy)
   if (curbuf->b_ml.ml_mfp == NULL && open_buffer(FALSE, NULL, 0) == FAIL)
     return FAIL;
 
+  bool readlen = true;
+
   if (copy) {
     line = vim_strsave(line);
   }
-  if (curbuf->b_ml.ml_line_lnum != lnum)            /* other line buffered */
+  if (curbuf->b_ml.ml_line_lnum != lnum) {           /* other line buffered */
     ml_flush_line(curbuf);                          /* flush it */
-  else if (curbuf->b_ml.ml_flags & ML_LINE_DIRTY)   /* same line allocated */
+  } else if (curbuf->b_ml.ml_flags & ML_LINE_DIRTY) {  /* same line allocated */
+    // TODO FIXME: see other "TODO FIXME"
+    curbuf->deleted_bytes += STRLEN(curbuf->b_ml.ml_line_ptr)+1;
     xfree(curbuf->b_ml.ml_line_ptr);             /* free it */
+    readlen = false; // already read it.
+  }
+
+  if (readlen) {
+    if (true) { // TODO: buffer updates active
+      curbuf->deleted_bytes += STRLEN(ml_get_buf(curbuf, lnum, false))+1;
+    }
+  }
+
   curbuf->b_ml.ml_line_ptr = line;
   curbuf->b_ml.ml_line_lnum = lnum;
   curbuf->b_ml.ml_flags = (curbuf->b_ml.ml_flags | ML_LINE_DIRTY) & ~ML_EMPTY;
@@ -2491,6 +2504,7 @@ static int ml_delete_int(buf_T *buf, linenr_T lnum, bool message)
   else
     line_size = ((dp->db_index[idx - 1]) & DB_INDEX_MASK) - line_start;
 
+  buf->deleted_bytes += line_size;
 
   /*
    * special case: If there is only one line in the data block it becomes empty.
@@ -2676,6 +2690,13 @@ void ml_clearmarked(void)
   return;
 }
 
+size_t ml_flush_deleted_bytes(buf_T *buf)
+{
+  size_t ret = buf->deleted_bytes;
+  buf->deleted_bytes = 0;
+  return ret;
+}
+
 /*
  * flush ml_line if necessary
  */
@@ -2704,6 +2725,8 @@ static void ml_flush_line(buf_T *buf)
       return;
     entered = TRUE;
 
+    buf->flush_count++;
+
     lnum = buf->b_ml.ml_line_lnum;
     new_line = buf->b_ml.ml_line_ptr;
 

src/nvim/misc1.c

@@ -1685,6 +1685,7 @@ int del_bytes(colnr_T count, bool fixpos_arg, bool use_delcombine)
   bool was_alloced = ml_line_alloced();     // check if oldp was allocated
   char_u *newp;
   if (was_alloced) {
+    curbuf->deleted_bytes += (size_t)oldlen+1;
     newp = oldp;                            // use same allocated memory
   } else {                                  // need to allocate a new line
     newp = xmalloc((size_t)(oldlen + 1 - count));

test/functional/lua/buffer_updates_spec.lua

@@ -5,6 +5,8 @@ local command = helpers.command
 local meths = helpers.meths
 local clear = helpers.clear
 local eq = helpers.eq
+local exec_lua = helpers.exec_lua
+local feed = helpers.feed
 
 local origlines = {"original line 1",
                    "original line 2",
@@ -16,7 +18,7 @@ local origlines = {"original line 1",
 describe('lua: buffer event callbacks', function()
   before_each(function()
     clear()
-    meths.execute_lua([[
+    exec_lua([[
       local events = {}
 
       function test_register(bufnr, id, changedtick)
@@ -38,55 +40,101 @@ describe('lua: buffer event callbacks', function()
         events = {}
         return ret_events
       end
-    ]], {})
+    ]])
   end)
 
-  it('works', function()
+
+  -- verifying the sizes with nvim_buf_get_offset is nice (checks we cannot
+  -- assert the wrong thing), but masks errors with unflushed lines (as
+  -- nvim_buf_get_offset forces a flush of the memline). To be safe run the
+  -- test both ways.
+  local function check(verify)
+    local lastsize
     meths.buf_set_lines(0, 0, -1, true, origlines)
-    meths.execute_lua("return test_register(...)", {0, "test1"})
+    if verify then
+      lastsize = meths.buf_get_offset(0, meths.buf_line_count(0))
+    end
+    exec_lua("return test_register(...)", 0, "test1")
     local tick = meths.buf_get_changedtick(0)
 
+    local verify_name = "test1"
+    local function check_events(expected)
+      local events = exec_lua("return get_events(...)" )
+      eq(expected, events)
+      if verify then
+        for _, event in ipairs(events) do
+          if event[1] == verify_name and event[2] == "lines" then
+            local startline, endline = event[5], event[7]
+            local newrange = meths.buf_get_offset(0, endline) - meths.buf_get_offset(0, startline)
+            local newsize = meths.buf_get_offset(0, meths.buf_line_count(0))
+            local oldrange = newrange + lastsize - newsize
+            eq(oldrange, event[8])
+            lastsize = newsize
+          end
+        end
+      end
+    end
+
     command('normal! GyyggP')
     tick = tick + 1
-    eq({{ "test1", "lines", 1, tick, 0, 0, 1 }},
+    check_events({{ "test1", "lines", 1, tick, 0, 0, 1, 0}})
-       meths.execute_lua("return get_events(...)", {}))
 
     meths.buf_set_lines(0, 3, 5, true, {"changed line"})
     tick = tick + 1
-    eq({{ "test1", "lines", 1, tick, 3, 5, 4 }},
+    check_events({{ "test1", "lines", 1, tick, 3, 5, 4, 32 }})
-       meths.execute_lua("return get_events(...)", {}))
 
-    meths.execute_lua("return test_register(...)", {0, "test2", true})
+    exec_lua("return test_register(...)", 0, "test2", true)
     tick = tick + 1
     command('undo')
 
     -- plugins can opt in to receive changedtick events, or choose
     -- to only recieve actual changes.
-    eq({{ "test1", "lines", 1, tick, 3, 4, 5 },
+    check_events({{ "test1", "lines", 1, tick, 3, 4, 5, 13 },
-        { "test2", "lines", 1, tick, 3, 4, 5 },
+        { "test2", "lines", 1, tick, 3, 4, 5, 13 },
-        { "test2", "changedtick", 1, tick+1 } },
+        { "test2", "changedtick", 1, tick+1 } })
-       meths.execute_lua("return get_events(...)", {}))
     tick = tick + 1
 
     -- simulate next callback returning true
-    meths.execute_lua("test_unreg = 'test1'", {})
+    exec_lua("test_unreg = 'test1'")
 
     meths.buf_set_lines(0, 6, 7, true, {"x1","x2","x3"})
     tick = tick + 1
 
     -- plugins can opt in to receive changedtick events, or choose
     -- to only recieve actual changes.
-    eq({{ "test1", "lines", 1, tick, 6, 7, 9 },
+    check_events({{ "test1", "lines", 1, tick, 6, 7, 9, 16 },
-        { "test2", "lines", 1, tick, 6, 7, 9 }},
+        { "test2", "lines", 1, tick, 6, 7, 9, 16 }})
-       meths.execute_lua("return get_events(...)", {}))
+
+    verify_name = "test2"
 
     meths.buf_set_lines(0, 1, 1, true, {"added"})
     tick = tick + 1
-    eq({{ "test2", "lines", 1, tick, 1, 1, 2 }},
+    check_events({{ "test2", "lines", 1, tick, 1, 1, 2, 0 }})
-       meths.execute_lua("return get_events(...)", {}))
+
+    feed('wix')
+    tick = tick + 1
+    check_events({{ "test2", "lines", 1, tick, 4, 5, 5, 16 }})
+
+    -- check hot path for multiple insert
+    feed('yz')
+    tick = tick + 1
+    check_events({{ "test2", "lines", 1, tick, 4, 5, 5, 17 }})
+
+    feed('<bs>')
+    tick = tick + 1
+    check_events({{ "test2", "lines", 1, tick, 4, 5, 5, 19 }})
+
+    feed('<esc>')
 
     command('bwipe!')
-    eq({{ "test2", "detach", 1 }},
+    check_events({{ "test2", "detach", 1 }})
-       meths.execute_lua("return get_events(...)", {}))
+   end
+
+  it('works', function()
+    check(false)
+  end)
+
+  it('works with verify', function()
+    check(true)
   end)
 end)