fix(api): on_bytes gets stale data on :substitute #36487

Problem: `extmark_splice()` was being called before `ml_replace()`,
which caused the on_bytes callback to be invoked with the old buffer
text instead of the new text.

Solution: store metadata for each match in a growing array, call
`ml_replace()` once to update the buffer, then call `extmark_splice()`
once per match.

Closes https://github.com/neovim/neovim/issues/36370.

(cherry picked from commit 7da4d6abe2)

src/nvim/ex_cmds.c

@@ -3552,6 +3552,25 @@ static int do_sub(exarg_T *eap, const proftime_T timeout, const int cmdpreview_n
       bool skip_match = false;
       linenr_T sub_firstlnum;           // nr of first sub line
 
+      // Track where substitutions started (set once per line).
+      linenr_T lnum_start = 0;
+
+      // Track per-line data for each match.
+      // Will be sent as a batch to `extmark_splice` after the substitution is done.
+      typedef struct {
+        int start_col;         // Position in new text where replacement goes
+        lpos_T start;          // Match start position in original text
+        lpos_T end;            // Match end position in original text
+        int matchcols;         // Columns deleted from original text
+        bcount_t matchbytes;   // Bytes deleted from original text
+        int subcols;           // Columns in replacement text
+        bcount_t subbytes;     // Bytes in replacement text
+        linenr_T lnum_before;  // Line number before this substitution
+        linenr_T lnum_after;   // Line number after this substitution
+      } LineData;
+
+      kvec_t(LineData) line_matches = KV_INITIAL_VALUE;
+
       // The new text is build up step by step, to avoid too much
       // copying.  There are these pieces:
       // sub_firstline  The old text, unmodified.
@@ -3903,7 +3922,7 @@ static int do_sub(exarg_T *eap, const proftime_T timeout, const int cmdpreview_n
         // 3. Substitute the string. During 'inccommand' preview only do this if
         //    there is a replace pattern.
         if (cmdpreview_ns <= 0 || has_second_delim) {
-          linenr_T lnum_start = lnum;  // save the start lnum
+          lnum_start = lnum;  // save the start lnum
           int save_ma = curbuf->b_p_ma;
           int save_sandbox = sandbox;
           if (subflags.do_count) {
@@ -3993,6 +4012,9 @@ static int do_sub(exarg_T *eap, const proftime_T timeout, const int cmdpreview_n
           }
           replaced_bytes += end.col - start.col;
 
+          // Save the line number before processing newlines.
+          linenr_T lnum_before_newlines = lnum;
+
           // Now the trick is to replace CTRL-M chars with a real line
           // break.  This would make it impossible to insert a CTRL-M in
           // the text.  The line break can be avoided by preceding the
@@ -4044,9 +4066,18 @@ static int do_sub(exarg_T *eap, const proftime_T timeout, const int cmdpreview_n
             u_save_cursor();
             did_save = true;
           }
-          extmark_splice(curbuf, (int)lnum_start - 1, start_col,
+
-                         end.lnum - start.lnum, matchcols, replaced_bytes,
+          // Store extmark data for this match.
-                         lnum - lnum_start, subcols, sublen - 1, kExtmarkUndo);
+          LineData *data = kv_pushp(line_matches);
+          data->start_col = start_col;
+          data->start = start;
+          data->end = end;
+          data->matchcols = matchcols;
+          data->matchbytes = replaced_bytes;
+          data->subcols = subcols;
+          data->subbytes = sublen - 1;
+          data->lnum_before = lnum_before_newlines;
+          data->lnum_after = lnum;
         }
 
         // 4. If subflags.do_all is set, find next match.
@@ -4097,6 +4128,21 @@ skip:
             }
             ml_replace(lnum, new_start, true);
 
+            // Call extmark_splice for each match on this line.
+            for (size_t match_idx = 0; match_idx < kv_size(line_matches); match_idx++) {
+              LineData *match = &kv_A(line_matches, match_idx);
+
+              extmark_splice(curbuf, (int)match->lnum_before - 1, match->start_col,
+                             match->end.lnum - match->start.lnum, match->matchcols,
+                             match->matchbytes,
+                             match->lnum_after - match->lnum_before,
+                             match->subcols,
+                             match->subbytes, kExtmarkUndo);
+            }
+
+            // Reset the match data for the next line.
+            kv_size(line_matches) = 0;
+
             if (nmatch_tl > 0) {
               // Matched lines have now been substituted and are
               // useless, delete them.  The part after the match
@@ -4192,6 +4238,7 @@ skip:
       }
       xfree(new_start);              // for when substitute was cancelled
       XFREE_CLEAR(sub_firstline);    // free the copy of the original line
+      kv_destroy(line_matches);      // clean up match data
     }
 
     line_breakcheck();

test/functional/lua/buffer_updates_spec.lua

@@ -1208,6 +1208,255 @@ describe('lua: nvim_buf_attach on_bytes', function()
       }
     end)
 
+    it('on_bytes sees modified buffer after substitute', function()
+      api.nvim_buf_set_lines(0, 0, -1, true, { 'Hello' })
+
+      local buffer_lines = exec_lua(function()
+        local lines
+        vim.api.nvim_buf_attach(0, false, {
+          on_bytes = function()
+            lines = vim.api.nvim_buf_get_lines(0, 0, -1, true)
+          end,
+        })
+        vim.cmd('s/llo/y/')
+        return lines
+      end)
+
+      -- Make sure on_bytes is called after the buffer is modified.
+      eq({ 'Hey' }, buffer_lines)
+    end)
+
+    it('on_bytes called multiple times for multiple substitutions on same line', function()
+      api.nvim_buf_set_lines(0, 0, -1, true, { 'Hello Hello' })
+
+      local call_count, args = exec_lua(function()
+        local count = 0
+        local args = {}
+        vim.api.nvim_buf_attach(0, false, {
+          on_bytes = function(
+            _,
+            _,
+            _,
+            start_row,
+            start_col,
+            start_byte,
+            old_row,
+            old_col,
+            old_byte,
+            new_row,
+            new_col,
+            new_byte
+          )
+            count = count + 1
+            table.insert(args, {
+              start_row = start_row,
+              start_col = start_col,
+              start_byte = start_byte,
+              old_row = old_row,
+              old_col = old_col,
+              old_byte = old_byte,
+              new_row = new_row,
+              new_col = new_col,
+              new_byte = new_byte,
+              buffer_lines = vim.api.nvim_buf_get_lines(0, 0, -1, true),
+            })
+          end,
+        })
+        vim.cmd('s/llo/y/g')
+        return count, args
+      end)
+
+      -- Should be called twice, once for each match.
+      eq(2, call_count)
+
+      -- First match: "llo" at column 2 -> "y".
+      eq({
+        start_row = 0,
+        start_col = 2,
+        start_byte = 2,
+        old_row = 0,
+        old_col = 3,
+        old_byte = 3,
+        new_row = 0,
+        new_col = 1,
+        new_byte = 1,
+        buffer_lines = { 'Hey Hey' },
+      }, args[1])
+
+      -- Second match: "llo" at column 8 (in original) -> column 6 (after first substitution).
+      eq({
+        start_row = 0,
+        start_col = 6, -- Adjusted position after first substitution.
+        start_byte = 6,
+        old_row = 0,
+        old_col = 3,
+        old_byte = 3,
+        new_row = 0,
+        new_col = 1,
+        new_byte = 1,
+        buffer_lines = { 'Hey Hey' },
+      }, args[2])
+    end)
+
+    it('on_bytes called correctly for multi-line substitutions', function()
+      api.nvim_buf_set_lines(0, 0, -1, true, { 'foo bar', 'baz qux' })
+
+      local call_count, args = exec_lua(function()
+        local count = 0
+        local args = {}
+        vim.api.nvim_buf_attach(0, false, {
+          on_bytes = function(
+            _,
+            _,
+            _,
+            start_row,
+            start_col,
+            start_byte,
+            old_row,
+            old_col,
+            old_byte,
+            new_row,
+            new_col,
+            new_byte
+          )
+            count = count + 1
+            table.insert(args, {
+              start_row = start_row,
+              start_col = start_col,
+              start_byte = start_byte,
+              old_row = old_row,
+              old_col = old_col,
+              old_byte = old_byte,
+              new_row = new_row,
+              new_col = new_col,
+              new_byte = new_byte,
+              buffer_lines = vim.api.nvim_buf_get_lines(0, 0, -1, true),
+            })
+          end,
+        })
+        vim.cmd('s/bar/X\\rY/')
+        return count, args
+      end)
+
+      -- Should be called once for the substitution.
+      eq(1, call_count)
+
+      eq({
+        start_row = 0,
+        start_col = 4,
+        start_byte = 4,
+        old_row = 0,
+        old_col = 3,
+        old_byte = 3,
+        new_row = 1,
+        new_col = 1,
+        new_byte = 3,
+        buffer_lines = { 'foo X', 'Y', 'baz qux' },
+      }, args[1])
+    end)
+
+    it('on_bytes called multiple times for global substitution creating multiple lines', function()
+      api.nvim_buf_set_lines(0, 0, -1, true, { 'foo bar baz' })
+
+      local call_count, args = exec_lua(function()
+        local count = 0
+        local args = {}
+        vim.api.nvim_buf_attach(0, false, {
+          on_bytes = function(
+            _,
+            _,
+            _,
+            start_row,
+            start_col,
+            start_byte,
+            old_row,
+            old_col,
+            old_byte,
+            new_row,
+            new_col,
+            new_byte
+          )
+            count = count + 1
+            table.insert(args, {
+              start_row = start_row,
+              start_col = start_col,
+              start_byte = start_byte,
+              old_row = old_row,
+              old_col = old_col,
+              old_byte = old_byte,
+              new_row = new_row,
+              new_col = new_col,
+              new_byte = new_byte,
+              buffer_lines = vim.api.nvim_buf_get_lines(0, 0, -1, true),
+            })
+          end,
+        })
+        -- Global substitution with newlines in replacement.
+        vim.cmd([[s/ /\r/g]])
+        return count, args
+      end)
+
+      -- Should be called once per space replacement.
+      eq(2, call_count)
+
+      eq({
+        start_row = 0,
+        start_col = 3,
+        start_byte = 3,
+        old_row = 0,
+        old_col = 1,
+        old_byte = 1,
+        new_row = 1,
+        new_col = 0,
+        new_byte = 1,
+        buffer_lines = { 'foo', 'bar', 'baz' },
+      }, args[1])
+
+      eq({
+        start_row = 1,
+        start_col = 3,
+        start_byte = 7,
+        old_row = 0,
+        old_col = 1,
+        old_byte = 1,
+        new_row = 1,
+        new_col = 0,
+        new_byte = 1,
+        buffer_lines = { 'foo', 'bar', 'baz' },
+      }, args[2])
+    end)
+
+    it(
+      'no buffer update event is emitted while editing substitute command, only after confirmation',
+      function()
+        api.nvim_buf_set_lines(0, 0, -1, true, { 'Hello world', 'Hello Neovim' })
+
+        exec_lua(function()
+          _G.num_buffer_updates = 0
+          vim.api.nvim_buf_attach(0, false, {
+            on_bytes = function()
+              _G.num_buffer_updates = _G.num_buffer_updates + 1
+            end,
+          })
+        end)
+
+        -- Start typing the substitute command - no events should be emitted yet.
+        feed(':%s/Hello/Hi')
+        eq(0, exec_lua('return _G.num_buffer_updates'))
+
+        -- Continue editing the command - still no events.
+        feed('<BS><BS>Hey')
+        eq(0, exec_lua('return _G.num_buffer_updates'))
+
+        -- After confirming the substitution, two events should be emitted (one per line).
+        feed('<CR>')
+        eq(2, exec_lua('return _G.num_buffer_updates'))
+
+        -- Verify the buffer was actually modified.
+        eq({ 'Hey world', 'Hey Neovim' }, api.nvim_buf_get_lines(0, 0, -1, true))
+      end
+    )
+
     it('flushes delbytes on join', function()
       local check_events = setup_eventcheck(verify, { 'AAA', 'BBB', 'CCC' })