diff --git a/src/nvim/window.c b/src/nvim/window.c
index 7f38357921..bee863e88b 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -6946,13 +6946,14 @@ void scroll_to_fraction(win_T *wp, int prev_height)
       wp->w_wrow = line_size;
       if (wp->w_wrow >= wp->w_view_height
           && (wp->w_view_width - win_col_off(wp)) > 0) {
-        wp->w_skipcol += wp->w_view_width - win_col_off(wp);
+        // The cursor must be visible, override the scroll position.
+        colnr_T skipcol = wp->w_view_width - win_col_off(wp);
         wp->w_wrow--;
         while (wp->w_wrow >= wp->w_view_height) {
-          wp->w_skipcol += wp->w_view_width - win_col_off(wp)
-                           + win_col_off2(wp);
+          skipcol += wp->w_view_width - win_col_off(wp) + win_col_off2(wp);
           wp->w_wrow--;
         }
+        wp->w_skipcol = skipcol;
       }
     } else if (sline > 0) {
       while (sline > 0 && lnum > 1) {
@@ -7032,7 +7033,11 @@ void win_set_inner_size(win_T *wp, bool valid_cursor)
     // There is no point in adjusting the scroll position when exiting.  Some
     // values might be invalid.
     if (valid_cursor && !exiting && (*p_spk == 'c' || wp->w_floating)) {
-      wp->w_skipcol = 0;
+      // With 'smoothscroll' w_skipcol is the scroll position, keep it.
+      // Otherwise it only keeps the cursor visible and is computed again.
+      if (!wp->w_p_sms) {
+        wp->w_skipcol = 0;
+      }
       scroll_to_fraction(wp, prev_height);
     }
     redraw_later(wp, UPD_SOME_VALID);
diff --git a/test/functional/legacy/scroll_opt_spec.lua b/test/functional/legacy/scroll_opt_spec.lua
index d041d0e3c9..af720063f5 100644
--- a/test/functional/legacy/scroll_opt_spec.lua
+++ b/test/functional/legacy/scroll_opt_spec.lua
@@ -1245,7 +1245,6 @@ describe('smoothscroll', function()
                                               |
     ]])
     exec('set showtabline=2')
-    feed('<C-E>')
     screen:expect([[
       {5: }{100:2}{5:+ [No Name] }{2:                          }|
       {1:<<<}e text with some text with some text |
@@ -1261,7 +1260,7 @@ describe('smoothscroll', function()
                                               |
     ]])
     exec('set winbar=winbar')
-    feed('<C-w>k<C-E>')
+    feed('<C-w>k')
     screen:expect([[
       {5: }{100:2}{5:+ [No Name] }{2:                          }|
       {5:winbar                                  }|
diff --git a/test/old/testdir/test_scroll_opt.vim b/test/old/testdir/test_scroll_opt.vim
index fbf8e51e7d..c7959aee92 100644
--- a/test/old/testdir/test_scroll_opt.vim
+++ b/test/old/testdir/test_scroll_opt.vim
@@ -1310,6 +1310,30 @@ func Test_smoothscroll_next_topline()
   bwipe!
 endfunc
 
+func Test_smoothscroll_keep_skipcol()
+  call NewWindow(10, 40)
+  setlocal smoothscroll
+  call setline(1, ['abcde '->repeat(150)]->repeat(2))
+
+  exe "norm! 10\<C-E>"
+  redraw
+  let skipcol = winsaveview().skipcol
+  call assert_notequal(0, skipcol)
+
+  " Changing the height of the window must not reset the scroll position.
+  resize -3
+  resize +3
+  redraw
+  call assert_equal(skipcol, winsaveview().skipcol)
+
+  " Using the autocommand window changes the height as well.
+  call bufload(bufadd(''))
+  redraw
+  call assert_equal(skipcol, winsaveview().skipcol)
+
+  bwipe!
+endfunc
+
 func Test_smoothscroll_long_line_zb()
   call NewWindow(10, 40)
   call setline(1, 'abcde '->repeat(150))