diff --git a/src/nvim/move.c b/src/nvim/move.c
index a406163608..a484bbbb75 100644
--- a/src/nvim/move.c
+++ b/src/nvim/move.c
@@ -1335,56 +1335,55 @@ bool scrolldown(win_T *wp, linenr_T line_count, int byfold)
   hasFolding(wp, wp->w_topline, &wp->w_topline, NULL);
   validate_cursor(wp);            // w_wrow needs to be valid
   for (int todo = line_count; todo > 0; todo--) {
-    if (wp->w_topfill < win_get_fill(wp, wp->w_topline)
-        && wp->w_topfill < wp->w_view_height - 1) {
+    bool can_fill = wp->w_topfill < wp->w_view_height - 1
+                    && wp->w_topfill < win_get_fill(wp, wp->w_topline);
+    // break when at the very top
+    if (wp->w_topline == 1 && !can_fill && (!do_sms || wp->w_skipcol < width1)) {
+      break;
+    }
+    if (do_sms && wp->w_skipcol >= width1) {
+      // scroll a screen line down
+      if (wp->w_skipcol >= width1 + width2) {
+        wp->w_skipcol -= width2;
+      } else {
+        wp->w_skipcol -= width1;
+      }
+      redraw_later(wp, UPD_NOT_VALID);
+      done++;
+    } else if (can_fill) {
       wp->w_topfill++;
       done++;
     } else {
-      // break when at the very top
-      if (wp->w_topline == 1 && (!do_sms || wp->w_skipcol < width1)) {
-        break;
-      }
-      if (do_sms && wp->w_skipcol >= width1) {
-        // scroll a screen line down
-        if (wp->w_skipcol >= width1 + width2) {
-          wp->w_skipcol -= width2;
-        } else {
-          wp->w_skipcol -= width1;
+      // scroll a text line down
+      wp->w_topline--;
+      wp->w_skipcol = 0;
+      wp->w_topfill = 0;
+      // A sequence of folded lines only counts for one logical line
+      linenr_T first;
+      if (hasFolding(wp, wp->w_topline, &first, NULL)) {
+        done += !decor_conceal_line(wp, first - 1, false);
+        if (!byfold) {
+          todo -= wp->w_topline - first - 1;
         }
-        redraw_later(wp, UPD_NOT_VALID);
-        done++;
+        wp->w_botline -= wp->w_topline - first;
+        wp->w_topline = first;
+      } else if (decor_conceal_line(wp, wp->w_topline - 1, false)) {
+        todo++;
       } else {
-        // scroll a text line down
-        wp->w_topline--;
-        wp->w_skipcol = 0;
-        wp->w_topfill = 0;
-        // A sequence of folded lines only counts for one logical line
-        linenr_T first;
-        if (hasFolding(wp, wp->w_topline, &first, NULL)) {
-          done += !decor_conceal_line(wp, first - 1, false);
-          if (!byfold) {
-            todo -= wp->w_topline - first - 1;
+        if (do_sms) {
+          int size = linetabsize_eol(wp, wp->w_topline);
+          if (size > width1) {
+            wp->w_skipcol = width1;
+            size -= width1;
+            redraw_later(wp, UPD_NOT_VALID);
           }
-          wp->w_botline -= wp->w_topline - first;
-          wp->w_topline = first;
-        } else if (decor_conceal_line(wp, wp->w_topline - 1, false)) {
-          todo++;
+          while (size > width2) {
+            wp->w_skipcol += width2;
+            size -= width2;
+          }
+          done++;
         } else {
-          if (do_sms) {
-            int size = linetabsize_eol(wp, wp->w_topline);
-            if (size > width1) {
-              wp->w_skipcol = width1;
-              size -= width1;
-              redraw_later(wp, UPD_NOT_VALID);
-            }
-            while (size > width2) {
-              wp->w_skipcol += width2;
-              size -= width2;
-            }
-            done++;
-          } else {
-            done += plines_win_nofill(wp, wp->w_topline, true);
-          }
+          done += plines_win_nofill(wp, wp->w_topline, true);
         }
       }
     }
diff --git a/test/functional/ui/decorations_spec.lua b/test/functional/ui/decorations_spec.lua
index 5d89bbdd80..3e990ed384 100644
--- a/test/functional/ui/decorations_spec.lua
+++ b/test/functional/ui/decorations_spec.lua
@@ -6677,6 +6677,33 @@ if (h->n_buckets < new_n_buckets) { // expand
       ]],
     })
   end)
+
+  it("not revealed before skipcol scrolling up with 'smoothscroll'", function()
+    api.nvim_set_option_value('smoothscroll', true, {})
+    api.nvim_buf_set_lines(0, 0, -1, false, { ('x'):rep(screen._width * 2) })
+    api.nvim_buf_set_extmark(0, ns, 0, 0, { virt_lines_above = true, virt_lines = { { { 'VIRT1' } } } } )
+    feed('<C-E>')
+    screen:expect([[
+      {1:<<<}xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx^x|
+      {1:~                                                 }|*10
+                                                        |
+    ]])
+    feed('<C-Y>')
+    screen:expect([[
+      xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx|
+      xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx^x|
+      {1:~                                                 }|*9
+                                                        |
+    ]])
+    feed('<C-Y>')
+    screen:expect([[
+      VIRT1                                             |
+      xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx|
+      xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx^x|
+      {1:~                                                 }|*8
+                                                        |
+    ]])
+    end)
 end)
 
 describe('decorations: signs', function()