fix(terminal): adjust marks when deleting scrollback lines (#36294)

This also fixes inconsistent scrolling behavior on terminal output when
cursor is in the middle of the buffer and the scrollback is full.

runtime/lua/vim/_defaults.lua

@@ -581,7 +581,13 @@ do
       if string.match(args.data.sequence, '^\027]133;A') then
         local lnum = args.data.cursor[1] ---@type integer
         if lnum >= 1 then
-          vim.api.nvim_buf_set_extmark(args.buf, nvim_terminal_prompt_ns, lnum - 1, 0, {})
+          vim.api.nvim_buf_set_extmark(
+            args.buf,
+            nvim_terminal_prompt_ns,
+            lnum - 1,
+            0,
+            { right_gravity = false }
+          )
         end
       end
     end,

src/nvim/api/buffer.c

@@ -429,7 +429,7 @@ void nvim_buf_set_lines(uint64_t channel_id, Buffer buffer, Integer start, Integ
     // changed range, and move any in the remainder of the buffer.
     linenr_T adjust = end > start ? MAXLNUM : 0;
     mark_adjust_buf(buf, (linenr_T)start, (linenr_T)(end - 1), adjust, (linenr_T)extra,
-                    true, true, kExtmarkNOOP);
+                    true, kMarkAdjustApi, kExtmarkNOOP);
 
     extmark_splice(buf, (int)start - 1, 0, (int)(end - start), 0,
                    deleted_bytes, (int)new_len, 0, inserted_bytes,
@@ -662,7 +662,7 @@ void nvim_buf_set_text(uint64_t channel_id, Buffer buffer, Integer start_row, In
     // Do not adjust any cursors. need to use column-aware logic (below)
     linenr_T adjust = end_row >= start_row ? MAXLNUM : 0;
     mark_adjust_buf(buf, (linenr_T)start_row, (linenr_T)end_row - 1, adjust, (linenr_T)extra,
-                    true, true, kExtmarkNOOP);
+                    true, kMarkAdjustApi, kExtmarkNOOP);
 
     extmark_splice(buf, (int)start_row - 1, (colnr_T)start_col,
                    (int)(end_row - start_row), col_extent, old_byte,

src/nvim/mark.c

@@ -1180,7 +1180,7 @@ void ex_changes(exarg_T *eap)
 void mark_adjust(linenr_T line1, linenr_T line2, linenr_T amount, linenr_T amount_after,
                  ExtmarkOp op)
 {
-  mark_adjust_buf(curbuf, line1, line2, amount, amount_after, true, false, op);
+  mark_adjust_buf(curbuf, line1, line2, amount, amount_after, true, kMarkAdjustNormal, op);
 }
 
 // mark_adjust_nofold() does the same as mark_adjust() but without adjusting
@@ -1191,11 +1191,11 @@ void mark_adjust(linenr_T line1, linenr_T line2, linenr_T amount, linenr_T amoun
 void mark_adjust_nofold(linenr_T line1, linenr_T line2, linenr_T amount, linenr_T amount_after,
                         ExtmarkOp op)
 {
-  mark_adjust_buf(curbuf, line1, line2, amount, amount_after, false, false, op);
+  mark_adjust_buf(curbuf, line1, line2, amount, amount_after, false, kMarkAdjustNormal, op);
 }
 
 void mark_adjust_buf(buf_T *buf, linenr_T line1, linenr_T line2, linenr_T amount,
-                     linenr_T amount_after, bool adjust_folds, bool by_api, ExtmarkOp op)
+                     linenr_T amount_after, bool adjust_folds, MarkAdjustMode mode, ExtmarkOp op)
 {
   int fnum = buf->b_fnum;
   linenr_T *lp;
@@ -1205,6 +1205,9 @@ void mark_adjust_buf(buf_T *buf, linenr_T line1, linenr_T line2, linenr_T amount
     return;
   }
 
+  bool by_api = mode == kMarkAdjustApi;
+  bool by_term = mode == kMarkAdjustTerm;
+
   if ((cmdmod.cmod_flags & CMOD_LOCKMARKS) == 0) {
     // named marks, lower case and upper case
     for (int i = 0; i < NMARKS; i++) {
@@ -1305,7 +1308,7 @@ void mark_adjust_buf(buf_T *buf, linenr_T line1, linenr_T line2, linenr_T amount
 
       // topline and cursor position for windows with the same buffer
       // other than the current window
-      if (win != curwin || by_api) {
+      if (by_api || (by_term ? win->w_cursor.lnum < buf->b_ml.ml_line_count : win != curwin)) {
         if (win->w_topline >= line1 && win->w_topline <= line2) {
           if (amount == MAXLNUM) {                  // topline is deleted
             if (by_api && amount_after > line1 - line2 - 1) {
@@ -1327,7 +1330,7 @@ void mark_adjust_buf(buf_T *buf, linenr_T line1, linenr_T line2, linenr_T amount
           win->w_topfill = 0;
         }
       }
-      if (win != curwin && !by_api) {
+      if (!by_api && (by_term ? win->w_cursor.lnum < buf->b_ml.ml_line_count : win != curwin)) {
         if (win->w_cursor.lnum >= line1 && win->w_cursor.lnum <= line2) {
           if (amount == MAXLNUM) {         // line with cursor is deleted
             if (line1 <= 1) {

src/nvim/mark_defs.h

@@ -39,6 +39,13 @@ typedef enum {
   kMarkAllNoResolve,  ///< Return all types of marks but don't resolve fnum (global marks).
 } MarkGet;
 
+/// Options when adjusting marks
+typedef enum {
+  kMarkAdjustNormal,  ///< Normal mode commands, etc.
+  kMarkAdjustApi,     ///< Changing lines from the API
+  kMarkAdjustTerm,    ///< Terminal scrollback
+} MarkAdjustMode;
+
 /// Number of possible numbered global marks
 #define EXTRA_MARKS     ('9' - '0' + 1)
 

src/nvim/terminal.c

@@ -74,6 +74,7 @@
 #include "nvim/macros_defs.h"
 #include "nvim/main.h"
 #include "nvim/map_defs.h"
+#include "nvim/mark.h"
 #include "nvim/mbyte.h"
 #include "nvim/memline.h"
 #include "nvim/memory.h"
@@ -159,10 +160,11 @@ struct terminal {
   // window height has increased) and must be deleted from the terminal buffer
   int sb_pending;
   size_t sb_deleted;                // Lines deleted from sb_buffer.
+  size_t sb_deleted_last;           // Value of sb_deleted on last refresh_scrollback()
 
   char *title;     // VTermStringFragment buffer
-  size_t title_len;    // number of rows pushed to sb_buffer
+  size_t title_len;
-  size_t title_size;   // sb_buffer size
+  size_t title_size;
 
   // buf_T instance that acts as a "drawing surface" for libvterm
   // we can't store a direct reference to the buffer because the
@@ -2220,6 +2222,8 @@ static void adjust_scrollback(Terminal *term, buf_T *buf)
       term->sb_current--;
       xfree(term->sb_buffer[term->sb_current]);
     }
+    mark_adjust_buf(buf, 1, (linenr_T)diff, MAXLNUM, -(linenr_T)diff, true,
+                    kMarkAdjustTerm, kExtmarkUndo);
     deleted_lines_buf(buf, 1, (linenr_T)diff);
   }
 
@@ -2235,6 +2239,11 @@ static void adjust_scrollback(Terminal *term, buf_T *buf)
 // Refresh the scrollback of an invalidated terminal.
 static void refresh_scrollback(Terminal *term, buf_T *buf)
 {
+  linenr_T deleted = (linenr_T)(term->sb_deleted - term->sb_deleted_last);
+  deleted = MIN(deleted, buf->b_ml.ml_line_count);
+  mark_adjust_buf(buf, 1, deleted, MAXLNUM, -deleted, true, kMarkAdjustTerm, kExtmarkUndo);
+  term->sb_deleted_last = term->sb_deleted;
+
   int width, height;
   vterm_get_size(term->vt, &height, &width);
 

test/functional/terminal/mouse_spec.lua

@@ -227,6 +227,15 @@ describe(':terminal mouse', function()
       it('will forward mouse clicks to the program with the correct even if set nu', function()
         skip(is_os('win'))
         command('set number')
+        screen:expect([[
+          {121: 11 }line28                                        |
+          {121: 12 }line29                                        |
+          {121: 13 }line30                                        |
+          {121: 14 }mouse enabled                                 |
+          {121: 15 }rows: 6, cols: 46                             |
+          {121: 16 }^                                              |
+          {5:-- TERMINAL --}                                    |
+        ]])
         -- When the display area such as a number is clicked, it returns to the
         -- normal mode.
         feed('<LeftMouse><3,0>')

test/functional/terminal/scrollback_spec.lua

@@ -5,6 +5,7 @@ local tt = require('test.functional.testterm')
 
 local clear, eq = n.clear, t.eq
 local feed, testprg = n.feed, n.testprg
+local fn = n.fn
 local eval = n.eval
 local command = n.command
 local poke_eventloop = n.poke_eventloop
@@ -25,6 +26,18 @@ describe(':terminal scrollback', function()
     screen = tt.setup_screen(nil, nil, 30)
   end)
 
+  local function feed_new_lines_and_wait(count)
+    local lines = {}
+    for i = 1, count do
+      table.insert(lines, 'new_line' .. tostring(i))
+    end
+    table.insert(lines, '')
+    feed_data(lines)
+    retry(nil, 1000, function()
+      eq({ 'new_line' .. tostring(count), '' }, api.nvim_buf_get_lines(0, -3, -1, true))
+    end)
+  end
+
   describe('when the limit is exceeded', function()
     before_each(function()
       local lines = {}
@@ -56,6 +69,108 @@ describe(':terminal scrollback', function()
                                       |
       ]])
     end)
+
+    describe('and cursor on non-last row in screen', function()
+      before_each(function()
+        feed([[<C-\><C-N>M$]])
+        fn.setpos("'m", { 0, 13, 4, 0 })
+        local ns = api.nvim_create_namespace('test')
+        api.nvim_buf_set_extmark(0, ns, 12, 0, { end_col = 6, hl_group = 'ErrorMsg' })
+        screen:expect([[
+          line26                        |
+          line27                        |
+          {101:line2^8}                        |
+          line29                        |
+          line30                        |
+                                        |*2
+        ]])
+      end)
+
+      it("when outputting fewer than 'scrollback' lines", function()
+        feed_new_lines_and_wait(6)
+        screen:expect([[
+          line26                        |
+          line27                        |
+          {101:line2^8}                        |
+          line29                        |
+          line30                        |
+          new_line1                     |
+                                        |
+        ]])
+        eq({ 0, 7, 4, 0 }, fn.getpos("'m"))
+        eq({ 0, 7, 6, 0 }, fn.getpos('.'))
+      end)
+
+      it("when outputting more than 'scrollback' lines", function()
+        feed_new_lines_and_wait(11)
+        screen:expect([[
+          line27                        |
+          {101:line2^8}                        |
+          line29                        |
+          line30                        |
+          new_line1                     |
+          new_line2                     |
+                                        |
+        ]])
+        eq({ 0, 2, 4, 0 }, fn.getpos("'m"))
+        eq({ 0, 2, 6, 0 }, fn.getpos('.'))
+      end)
+
+      it('when outputting more lines than whole buffer', function()
+        feed_new_lines_and_wait(20)
+        screen:expect([[
+          ^new_line6                     |
+          new_line7                     |
+          new_line8                     |
+          new_line9                     |
+          new_line10                    |
+          new_line11                    |
+                                        |
+        ]])
+        eq({ 0, 0, 0, 0 }, fn.getpos("'m"))
+        eq({ 0, 1, 1, 0 }, fn.getpos('.'))
+      end)
+    end)
+
+    describe('and cursor on scrollback row #12651', function()
+      before_each(function()
+        feed([[<C-\><C-N>Hk$]])
+        fn.setpos("'m", { 0, 10, 4, 0 })
+        local ns = api.nvim_create_namespace('test')
+        api.nvim_buf_set_extmark(0, ns, 9, 0, { end_col = 6, hl_group = 'ErrorMsg' })
+        screen:expect([[
+          {101:line2^5}                        |
+          line26                        |
+          line27                        |
+          line28                        |
+          line29                        |
+          line30                        |
+                                        |
+        ]])
+      end)
+
+      it("when outputting fewer than 'scrollback' lines", function()
+        feed_new_lines_and_wait(6)
+        screen:expect_unchanged()
+        eq({ 0, 4, 4, 0 }, fn.getpos("'m"))
+        eq({ 0, 4, 6, 0 }, fn.getpos('.'))
+      end)
+
+      it("when outputting more than 'scrollback' lines", function()
+        feed_new_lines_and_wait(11)
+        screen:expect([[
+          ^line27                        |
+          line28                        |
+          line29                        |
+          line30                        |
+          new_line1                     |
+          new_line2                     |
+                                        |
+        ]])
+        eq({ 0, 0, 0, 0 }, fn.getpos("'m"))
+        eq({ 0, 1, 1, 0 }, fn.getpos('.'))
+      end)
+    end)
   end)
 
   describe('with cursor at last row', function()
@@ -70,6 +185,43 @@ describe(':terminal scrollback', function()
         ^                              |
         {5:-- TERMINAL --}                |
       ]])
+      fn.setpos("'m", { 0, 3, 4, 0 })
+      local ns = api.nvim_create_namespace('test')
+      api.nvim_buf_set_extmark(0, ns, 2, 0, { end_col = 5, hl_group = 'ErrorMsg' })
+      screen:expect([[
+        tty ready                     |
+        line1                         |
+        {101:line2}                         |
+        line3                         |
+        line4                         |
+        ^                              |
+        {5:-- TERMINAL --}                |
+      ]])
+    end)
+
+    it("when outputting more than 'scrollback' lines in Normal mode", function()
+      feed([[<C-\><C-N>]])
+      feed_new_lines_and_wait(11)
+      screen:expect([[
+        new_line7                     |
+        new_line8                     |
+        new_line9                     |
+        new_line10                    |
+        new_line11                    |
+        ^                              |
+                                      |
+      ]])
+      feed('gg')
+      screen:expect([[
+        ^line1                         |
+        {101:line2}                         |
+        line3                         |
+        line4                         |
+        new_line1                     |
+        new_line2                     |
+                                      |
+      ]])
+      eq({ 0, 2, 4, 0 }, fn.getpos("'m"))
     end)
 
     describe('and 1 line is printed', function()
@@ -80,7 +232,7 @@ describe(':terminal scrollback', function()
       it('will hide the top line', function()
         screen:expect([[
           line1                         |
-          line2                         |
+          {101:line2}                         |
           line3                         |
           line4                         |
           line5                         |
@@ -88,32 +240,34 @@ describe(':terminal scrollback', function()
           {5:-- TERMINAL --}                |
         ]])
         eq(7, api.nvim_buf_line_count(0))
+        eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
       end)
 
       describe('and then 3 more lines are printed', function()
         before_each(function()
-          feed_data({ 'line6', 'line7', 'line8' })
+          feed_data({ 'line6', 'line7', 'line8', '' })
         end)
 
         it('will hide the top 4 lines', function()
           screen:expect([[
-            line3                         |
             line4                         |
             line5                         |
             line6                         |
             line7                         |
-            line8^                         |
+            line8                         |
+            ^                              |
             {5:-- TERMINAL --}                |
           ]])
+          eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
 
           feed('<c-\\><c-n>6k')
           screen:expect([[
-            ^line2                         |
+            ^line3                         |
-            line3                         |
             line4                         |
             line5                         |
             line6                         |
             line7                         |
+            line8                         |
                                           |
           ]])
 
@@ -121,7 +275,7 @@ describe(':terminal scrollback', function()
           screen:expect([[
             ^tty ready                     |
             line1                         |
-            line2                         |
+            {101:line2}                         |
             line3                         |
             line4                         |
             line5                         |
@@ -130,12 +284,12 @@ describe(':terminal scrollback', function()
 
           feed('G')
           screen:expect([[
-            line3                         |
             line4                         |
             line5                         |
             line6                         |
             line7                         |
-            ^line8                         |
+            line8                         |
+            ^                              |
                                           |
           ]])
         end)
@@ -147,13 +301,14 @@ describe(':terminal scrollback', function()
         feed([[<C-\><C-N>]])
         screen:try_resize(screen._width - 2, screen._height - 1)
         screen:expect([[
-          line2                       |
+          {101:line2}                       |
           line3                       |
           line4                       |
           rows: 5, cols: 28           |
           ^                            |
                                       |
         ]])
+        eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
       end
 
       it('will hide top line', will_hide_top_line)
@@ -172,13 +327,21 @@ describe(':terminal scrollback', function()
                                       |
           ]])
           eq(8, api.nvim_buf_line_count(0))
-          feed([[3k]])
+          eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
+          feed('3k')
           screen:expect([[
             ^line4                     |
             rows: 5, cols: 28         |
             rows: 3, cols: 26         |
                                       |
           ]])
+          feed('gg')
+          screen:expect([[
+            ^tty ready                 |
+            line1                     |
+            {101:line2}                     |
+                                      |
+          ]])
         end)
       end)
     end)
@@ -255,6 +418,18 @@ describe(':terminal scrollback', function()
         ^                              |
         {5:-- TERMINAL --}                |
       ]])
+      fn.setpos("'m", { 0, 3, 4, 0 })
+      local ns = api.nvim_create_namespace('test')
+      api.nvim_buf_set_extmark(0, ns, 2, 0, { end_col = 5, hl_group = 'ErrorMsg' })
+      screen:expect([[
+        tty ready                     |
+        line1                         |
+        {101:line2}                         |
+        line3                         |
+        line4                         |
+        ^                              |
+        {5:-- TERMINAL --}                |
+      ]])
       screen:try_resize(screen._width, screen._height - 3)
       screen:expect([[
         line4                         |
@@ -281,6 +456,7 @@ describe(':terminal scrollback', function()
           ^                              |
           {5:-- TERMINAL --}                |
         ]])
+        eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
       end
 
       it('will pop 1 line and then push it back', pop_then_push)
@@ -294,7 +470,7 @@ describe(':terminal scrollback', function()
 
         local function pop3_then_push1()
           screen:expect([[
-            line2                         |
+            {101:line2}                         |
             line3                         |
             line4                         |
             rows: 3, cols: 30             |
@@ -304,11 +480,12 @@ describe(':terminal scrollback', function()
             {5:-- TERMINAL --}                |
           ]])
           eq(9, api.nvim_buf_line_count(0))
+          eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
           feed('<c-\\><c-n>gg')
           screen:expect([[
             ^tty ready                     |
             line1                         |
-            line2                         |
+            {101:line2}                         |
             line3                         |
             line4                         |
             rows: 3, cols: 30             |
@@ -330,7 +507,7 @@ describe(':terminal scrollback', function()
             screen:expect([[
               tty ready                     |
               line1                         |
-              line2                         |
+              {101:line2}                         |
               line3                         |
               line4                         |
               rows: 3, cols: 30             |
@@ -344,6 +521,7 @@ describe(':terminal scrollback', function()
             -- since there's an empty line after the cursor, the buffer line
             -- count equals the terminal screen height
             eq(11, api.nvim_buf_line_count(0))
+            eq({ 0, 3, 4, 0 }, fn.getpos("'m"))
           end)
         end)
       end)
@@ -484,22 +662,36 @@ describe("'scrollback' option", function()
     table.insert(lines, '')
     feed_data(lines)
     screen:expect([[
-        line26                        |
+      line26                        |
-        line27                        |
+      line27                        |
-        line28                        |
+      line28                        |
-        line29                        |
+      line29                        |
-        line30                        |
+      line30                        |
-        ^                              |
+      ^                              |
-        {5:-- TERMINAL --}                |
+      {5:-- TERMINAL --}                |
-      ]])
+    ]])
+    local ns = api.nvim_create_namespace('test')
     local term_height = 6 -- Actual terminal screen height, not the scrollback
     -- Initial
     local scrollback = api.nvim_get_option_value('scrollback', {})
-    eq(scrollback + term_height, eval('line("$")'))
+    eq(scrollback + term_height, fn.line('$'))
+    n.fn.setpos("'m", { 0, scrollback + 1, 4, 0 })
+    api.nvim_buf_set_extmark(0, ns, scrollback, 0, { end_col = 6, hl_group = 'ErrorMsg' })
+    screen:expect([[
+      {101:line26}                        |
+      line27                        |
+      line28                        |
+      line29                        |
+      line30                        |
+      ^                              |
+      {5:-- TERMINAL --}                |
+    ]])
     -- Reduction
     scrollback = scrollback - 2
     api.nvim_set_option_value('scrollback', scrollback, {})
-    eq(scrollback + term_height, eval('line("$")'))
+    eq(scrollback + term_height, fn.line('$'))
+    screen:expect_unchanged()
+    eq({ 0, scrollback + 1, 4, 0 }, n.fn.getpos("'m"))
   end)
 
   it('defaults to 10000 in :terminal buffers', function()