diff --git a/src/nvim/autocmd.c b/src/nvim/autocmd.c
index 183750857b..3f7de39caf 100644
--- a/src/nvim/autocmd.c
+++ b/src/nvim/autocmd.c
@@ -1297,7 +1297,6 @@ void aucmd_prepbuf(aco_save_T *aco, buf_T *buf)
 
   aco->save_curwin_handle = curwin->handle;
   aco->save_prevwin_handle = prevwin == NULL ? 0 : prevwin->handle;
-  aco->save_State = State;
   if (bt_prompt(curbuf)) {
     aco->save_prompt_insert = curbuf->b_prompt_insert;
   }
@@ -1380,13 +1379,6 @@ void aucmd_restbuf(aco_save_T *aco)
     }
 win_found:
     curbuf->b_nwindows--;
-    const bool save_stop_insert_mode = stop_insert_mode;
-    // May need to stop Insert mode if we were in a prompt buffer.
-    leaving_window(curwin);
-    // Do not stop Insert mode when already in Insert mode before.
-    if (aco->save_State & MODE_INSERT) {
-      stop_insert_mode = save_stop_insert_mode;
-    }
     // Remove the window.
     win_remove(curwin, NULL);
     pmap_del(int)(&window_handles, curwin->handle, NULL);
diff --git a/src/nvim/autocmd_defs.h b/src/nvim/autocmd_defs.h
index 115801cffe..f11a40743c 100644
--- a/src/nvim/autocmd_defs.h
+++ b/src/nvim/autocmd_defs.h
@@ -20,7 +20,6 @@ typedef struct {
   char *tp_localdir;              ///< saved value of tp_localdir
   char *globaldir;                ///< saved value of globaldir
   bool save_VIsual_active;        ///< saved VIsual_active
-  int save_State;                 ///< saved State
   int save_prompt_insert;         ///< saved b_prompt_insert
 } aco_save_T;
 
diff --git a/src/nvim/window.c b/src/nvim/window.c
index 6179c33b65..b8284f3315 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -2492,7 +2492,9 @@ void leaving_window(win_T *const win)
   FUNC_ATTR_NONNULL_ALL
 {
   // Only matters for a prompt window.
-  if (!bt_prompt(win->w_buffer)) {
+  // Don't do mode changes for a prompt buffer in an autocommand window, as
+  // it's only used temporarily during an autocommand.
+  if (!bt_prompt(win->w_buffer) || is_aucmd_win(win)) {
     return;
   }
 
@@ -2519,7 +2521,9 @@ void entering_window(win_T *const win)
   FUNC_ATTR_NONNULL_ALL
 {
   // Only matters for a prompt window.
-  if (!bt_prompt(win->w_buffer)) {
+  // Don't do mode changes for a prompt buffer in an autocommand window, as
+  // it's only used temporarily during an autocommand.
+  if (!bt_prompt(win->w_buffer) || is_aucmd_win(win)) {
     return;
   }
 
diff --git a/test/functional/legacy/prompt_buffer_spec.lua b/test/functional/legacy/prompt_buffer_spec.lua
index 7e704808a3..550853d06b 100644
--- a/test/functional/legacy/prompt_buffer_spec.lua
+++ b/test/functional/legacy/prompt_buffer_spec.lua
@@ -201,10 +201,18 @@ describe('prompt buffer', function()
       endfunc
       call prompt_setcallback(bufnr(), function('s:TextEntered'))
 
-      func DoAppend()
+      func DoAppend(cmd_before = '')
+        exe a:cmd_before
         call appendbufline('prompt', '$', 'Test')
         return ''
       endfunc
+
+      autocmd User SwitchTabPages tabprevious | tabnext
+      func DoAutoAll(cmd_before = '')
+        exe a:cmd_before
+        doautoall User SwitchTabPages
+        return ''
+      endfunc
     ]])
     feed('asomething<CR>')
     eq('something', api.nvim_get_var('entered'))
@@ -218,6 +226,17 @@ describe('prompt buffer', function()
     eq({ mode = 'i', blocking = false }, api.nvim_get_mode())
     command('call DoAppend()')
     eq({ mode = 'i', blocking = false }, api.nvim_get_mode())
+    command("call DoAppend('stopinsert')")
+    eq({ mode = 'n', blocking = false }, api.nvim_get_mode())
+    command("call DoAppend('startreplace')")
+    eq({ mode = 'R', blocking = false }, api.nvim_get_mode())
+    feed('<Esc>')
+    command('tabnew')
+    eq({ mode = 'n', blocking = false }, api.nvim_get_mode())
+    command("call DoAutoAll('startinsert')")
+    eq({ mode = 'i', blocking = false }, api.nvim_get_mode())
+    command("call DoAutoAll('stopinsert')")
+    eq({ mode = 'n', blocking = false }, api.nvim_get_mode())
   end)
 
   -- oldtest: Test_prompt_leave_modify_hidden()
diff --git a/test/old/testdir/test_prompt_buffer.vim b/test/old/testdir/test_prompt_buffer.vim
index 9aa3b87e34..feb1dbb7f2 100644
--- a/test/old/testdir/test_prompt_buffer.vim
+++ b/test/old/testdir/test_prompt_buffer.vim
@@ -270,10 +270,18 @@ func Test_prompt_appending_while_hidden()
       endfunc
       call prompt_setcallback(bufnr(), function('s:TextEntered'))
 
-      func DoAppend()
+      func DoAppend(cmd_before = '')
+        exe a:cmd_before
         call appendbufline('prompt', '$', 'Test')
         return ''
       endfunc
+
+      autocmd User SwitchTabPages tabprevious | tabnext
+      func DoAutoAll(cmd_before = '')
+        exe a:cmd_before
+        doautoall User SwitchTabPages
+        return ''
+      endfunc
   END
   call writefile(script, 'XpromptBuffer', 'D')
 
@@ -284,18 +292,32 @@ func Test_prompt_appending_while_hidden()
   call TermWait(buf)
 
   call term_sendkeys(buf, "exit\<CR>")
-  call WaitForAssert({-> assert_notmatch('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, ":call DoAppend()\<CR>")
-  call WaitForAssert({-> assert_notmatch('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, "i")
-  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_match('^-- INSERT --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, "\<C-R>=DoAppend()\<CR>")
-  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_match('^-- INSERT --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, "\<C-R>=DoAppend('stopinsert')\<CR>")
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, ":call DoAppend('startreplace')\<CR>")
+  call WaitForAssert({-> assert_match('^-- REPLACE --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, "\<Esc>:tabnew\<CR>")
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, ":call DoAutoAll('startinsert')\<CR>")
+  call WaitForAssert({-> assert_match('^-- INSERT --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, "\<C-R>=DoAutoAll('stopinsert')\<CR>")
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
 
-  call term_sendkeys(buf, "\<Esc>")
   call StopVimInTerminal(buf)
 endfunc
 
@@ -324,16 +346,16 @@ func Test_prompt_leave_modify_hidden()
   call TermWait(buf)
 
   call term_sendkeys(buf, "a")
-  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_match('^-- INSERT --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, "w")
-  call WaitForAssert({-> assert_notmatch('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, "\<C-W>w")
-  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_match('^-- INSERT --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, "q")
-  call WaitForAssert({-> assert_notmatch('-- INSERT --', term_getline(buf, 10))})
+  call WaitForAssert({-> assert_notmatch('-- .* --', term_getline(buf, 10))})
 
   call term_sendkeys(buf, ":bwipe!\<CR>")
   call WaitForAssert({-> assert_equal('Leave', term_getline(buf, 2))})